@mbc-cqrs-serverless/import 1.2.0 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/constant/sqs.constant.d.ts +20 -0
- package/dist/constant/sqs.constant.js +12 -0
- package/dist/constant/sqs.constant.js.map +1 -0
- package/dist/event/csv-import.queue.event.handler.js +8 -2
- package/dist/event/csv-import.queue.event.handler.js.map +1 -1
- package/dist/event/csv-import.sfn.event.d.ts +17 -6
- package/dist/event/csv-import.sfn.event.handler.d.ts +6 -13
- package/dist/event/csv-import.sfn.event.handler.js +237 -85
- package/dist/event/csv-import.sfn.event.handler.js.map +1 -1
- package/dist/event/csv-import.sfn.event.js.map +1 -1
- package/dist/event/import.event.handler.d.ts +5 -3
- package/dist/event/import.event.handler.js +11 -6
- package/dist/event/import.event.handler.js.map +1 -1
- package/dist/event/import.queue.event.d.ts +3 -0
- package/dist/event/import.queue.event.handler.d.ts +6 -23
- package/dist/event/import.queue.event.handler.js +15 -139
- package/dist/event/import.queue.event.handler.js.map +1 -1
- package/dist/event/import.queue.event.js +25 -4
- package/dist/event/import.queue.event.js.map +1 -1
- package/dist/event/processor/csv-batch.processor.d.ts +10 -0
- package/dist/event/processor/csv-batch.processor.js +66 -0
- package/dist/event/processor/csv-batch.processor.js.map +1 -0
- package/dist/event/processor/single-import.processor.d.ts +12 -0
- package/dist/event/processor/single-import.processor.js +132 -0
- package/dist/event/processor/single-import.processor.js.map +1 -0
- package/dist/event/zip-import.queue.event.handler.js +9 -2
- package/dist/event/zip-import.queue.event.handler.js.map +1 -1
- package/dist/import.module.js +4 -0
- package/dist/import.module.js.map +1 -1
- package/package.json +7 -3
|
@@ -1,10 +1,12 @@
|
|
|
1
|
-
import { IEventHandler,
|
|
1
|
+
import { IEventHandler, SqsService } from '@mbc-cqrs-serverless/core';
|
|
2
|
+
import { ConfigService } from '@nestjs/config';
|
|
2
3
|
import { ImportService } from '../import.service';
|
|
3
4
|
import { ImportEvent } from './import.event';
|
|
4
5
|
export declare class ImportEventHandler implements IEventHandler<ImportEvent> {
|
|
5
|
-
private readonly
|
|
6
|
+
private readonly sqsService;
|
|
7
|
+
private readonly configService;
|
|
6
8
|
private readonly importService;
|
|
7
9
|
private readonly logger;
|
|
8
|
-
constructor(
|
|
10
|
+
constructor(sqsService: SqsService, configService: ConfigService, importService: ImportService);
|
|
9
11
|
execute(event: ImportEvent): Promise<any>;
|
|
10
12
|
}
|
|
@@ -13,26 +13,31 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
13
13
|
exports.ImportEventHandler = void 0;
|
|
14
14
|
const core_1 = require("@mbc-cqrs-serverless/core");
|
|
15
15
|
const common_1 = require("@nestjs/common");
|
|
16
|
+
const config_1 = require("@nestjs/config");
|
|
17
|
+
const sqs_constant_1 = require("../constant/sqs.constant");
|
|
16
18
|
const import_status_enum_1 = require("../enum/import-status.enum");
|
|
17
19
|
const import_service_1 = require("../import.service");
|
|
18
20
|
const import_event_1 = require("./import.event");
|
|
19
21
|
let ImportEventHandler = ImportEventHandler_1 = class ImportEventHandler {
|
|
20
|
-
constructor(
|
|
21
|
-
this.
|
|
22
|
+
constructor(sqsService, configService, importService) {
|
|
23
|
+
this.sqsService = sqsService;
|
|
24
|
+
this.configService = configService;
|
|
22
25
|
this.importService = importService;
|
|
23
26
|
this.logger = new common_1.Logger(ImportEventHandler_1.name);
|
|
24
27
|
}
|
|
25
28
|
async execute(event) {
|
|
26
|
-
this.
|
|
27
|
-
|
|
28
|
-
|
|
29
|
+
const queueUrl = this.configService.get('IMPORT_QUEUE_URL');
|
|
30
|
+
if (!queueUrl)
|
|
31
|
+
throw new Error('IMPORT_QUEUE_URL is not configured');
|
|
32
|
+
await this.sqsService.sendMessage(queueUrl, JSON.stringify({ action: sqs_constant_1.ACTION_SINGLE_IMPORT_PROCESS, ...event }));
|
|
29
33
|
return await this.importService.updateStatus(event.importKey, import_status_enum_1.ImportStatusEnum.QUEUED);
|
|
30
34
|
}
|
|
31
35
|
};
|
|
32
36
|
exports.ImportEventHandler = ImportEventHandler;
|
|
33
37
|
exports.ImportEventHandler = ImportEventHandler = ImportEventHandler_1 = __decorate([
|
|
34
38
|
(0, core_1.EventHandler)(import_event_1.ImportEvent),
|
|
35
|
-
__metadata("design:paramtypes", [core_1.
|
|
39
|
+
__metadata("design:paramtypes", [core_1.SqsService,
|
|
40
|
+
config_1.ConfigService,
|
|
36
41
|
import_service_1.ImportService])
|
|
37
42
|
], ImportEventHandler);
|
|
38
43
|
//# sourceMappingURL=import.event.handler.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"import.event.handler.js","sourceRoot":"","sources":["../../src/event/import.event.handler.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,oDAIkC;AAClC,2CAAuC;
|
|
1
|
+
{"version":3,"file":"import.event.handler.js","sourceRoot":"","sources":["../../src/event/import.event.handler.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,oDAIkC;AAClC,2CAAuC;AACvC,2CAA8C;AAE9C,2DAAuE;AACvE,mEAA6D;AAC7D,sDAAiD;AACjD,iDAA4C;AAGrC,IAAM,kBAAkB,0BAAxB,MAAM,kBAAkB;IAG7B,YACmB,UAAsB,EACtB,aAA4B,EAC5B,aAA4B;QAF5B,eAAU,GAAV,UAAU,CAAY;QACtB,kBAAa,GAAb,aAAa,CAAe;QAC5B,kBAAa,GAAb,aAAa,CAAe;QAL9B,WAAM,GAAG,IAAI,eAAM,CAAC,oBAAkB,CAAC,IAAI,CAAC,CAAA;IAM1D,CAAC;IAEJ,KAAK,CAAC,OAAO,CAAC,KAAkB;QAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAS,kBAAkB,CAAC,CAAA;QACnE,IAAI,CAAC,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QAEpE,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAC/B,QAAQ,EACR,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,2CAA4B,EAAE,GAAG,KAAK,EAAE,CAAC,CACnE,CAAA;QAED,OAAO,MAAM,IAAI,CAAC,aAAa,CAAC,YAAY,CAC1C,KAAK,CAAC,SAAS,EACf,qCAAgB,CAAC,MAAM,CACxB,CAAA;IACH,CAAC;CACF,CAAA;AAvBY,gDAAkB;6BAAlB,kBAAkB;IAD9B,IAAA,mBAAY,EAAC,0BAAW,CAAC;qCAKO,iBAAU;QACP,sBAAa;QACb,8BAAa;GANpC,kBAAkB,CAuB9B"}
|
|
@@ -12,7 +12,10 @@ export declare class ImportQueueEvent implements IEvent, SQSRecord {
|
|
|
12
12
|
eventSource: string;
|
|
13
13
|
eventSourceARN: string;
|
|
14
14
|
awsRegion: string;
|
|
15
|
+
private _payload?;
|
|
15
16
|
private _importEvent?;
|
|
16
17
|
fromSqsRecord(record: SQSRecord): ImportQueueEvent;
|
|
18
|
+
get payload(): any;
|
|
19
|
+
get isCsvBatch(): boolean;
|
|
17
20
|
get importEvent(): ImportEvent;
|
|
18
21
|
}
|
|
@@ -1,28 +1,11 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @file import.queue.event.handler.ts
|
|
3
|
-
* @description This handler is the core worker for processing individual import records.
|
|
4
|
-
* It listens for messages from the main SQS queue, finds the correct processing
|
|
5
|
-
* strategy for the record's table type, and executes them sequentially.
|
|
6
|
-
*/
|
|
7
1
|
import { IEventHandler } from '@mbc-cqrs-serverless/core';
|
|
8
|
-
import { ImportPublishMode } from '../constant';
|
|
9
|
-
import { ImportService } from '../import.service';
|
|
10
|
-
import { IProcessStrategy } from '../interface/processing-strategy.interface';
|
|
11
2
|
import { ImportQueueEvent } from './import.queue.event';
|
|
3
|
+
import { CsvBatchProcessor } from './processor/csv-batch.processor';
|
|
4
|
+
import { SingleImportProcessor } from './processor/single-import.processor';
|
|
12
5
|
export declare class ImportQueueEventHandler implements IEventHandler<ImportQueueEvent> {
|
|
13
|
-
private readonly
|
|
14
|
-
private readonly
|
|
15
|
-
private readonly publishModeMap;
|
|
6
|
+
private readonly csvBatchProcessor;
|
|
7
|
+
private readonly singleImportProcessor;
|
|
16
8
|
private readonly logger;
|
|
17
|
-
constructor(
|
|
18
|
-
execute(event: ImportQueueEvent): Promise<
|
|
19
|
-
/**
|
|
20
|
-
* Orchestrates the processing of a single import record.
|
|
21
|
-
*/
|
|
22
|
-
handleImport(event: ImportQueueEvent): Promise<any>;
|
|
23
|
-
/**
|
|
24
|
-
* Executes the full lifecycle (compare, map, save) for a single strategy.
|
|
25
|
-
* @returns The result of the create/update operation or a status message.
|
|
26
|
-
*/
|
|
27
|
-
private executeStrategy;
|
|
9
|
+
constructor(csvBatchProcessor: CsvBatchProcessor, singleImportProcessor: SingleImportProcessor);
|
|
10
|
+
execute(event: ImportQueueEvent): Promise<void>;
|
|
28
11
|
}
|
|
@@ -8,164 +8,40 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
|
|
|
8
8
|
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
9
|
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
10
|
};
|
|
11
|
-
var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
12
|
-
return function (target, key) { decorator(target, key, paramIndex); }
|
|
13
|
-
};
|
|
14
11
|
var ImportQueueEventHandler_1;
|
|
15
12
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
13
|
exports.ImportQueueEventHandler = void 0;
|
|
17
|
-
/**
|
|
18
|
-
* @file import.queue.event.handler.ts
|
|
19
|
-
* @description This handler is the core worker for processing individual import records.
|
|
20
|
-
* It listens for messages from the main SQS queue, finds the correct processing
|
|
21
|
-
* strategy for the record's table type, and executes them sequentially.
|
|
22
|
-
*/
|
|
23
14
|
const core_1 = require("@mbc-cqrs-serverless/core");
|
|
24
15
|
const common_1 = require("@nestjs/common");
|
|
25
|
-
const constant_1 = require("../constant");
|
|
26
|
-
const comparison_status_enum_1 = require("../enum/comparison-status.enum");
|
|
27
|
-
const import_status_enum_1 = require("../enum/import-status.enum");
|
|
28
|
-
const helpers_1 = require("../helpers");
|
|
29
|
-
const import_module_definition_1 = require("../import.module-definition");
|
|
30
|
-
const import_service_1 = require("../import.service");
|
|
31
16
|
const import_queue_event_1 = require("./import.queue.event");
|
|
17
|
+
const csv_batch_processor_1 = require("./processor/csv-batch.processor");
|
|
18
|
+
const single_import_processor_1 = require("./processor/single-import.processor");
|
|
32
19
|
let ImportQueueEventHandler = ImportQueueEventHandler_1 = class ImportQueueEventHandler {
|
|
33
|
-
constructor(
|
|
34
|
-
this.
|
|
35
|
-
this.
|
|
36
|
-
this.publishModeMap = publishModeMap;
|
|
20
|
+
constructor(csvBatchProcessor, singleImportProcessor) {
|
|
21
|
+
this.csvBatchProcessor = csvBatchProcessor;
|
|
22
|
+
this.singleImportProcessor = singleImportProcessor;
|
|
37
23
|
this.logger = new common_1.Logger(ImportQueueEventHandler_1.name);
|
|
38
24
|
}
|
|
39
25
|
async execute(event) {
|
|
40
|
-
const importEntity = event.importEvent.importEntity;
|
|
41
|
-
if (!importEntity.id.startsWith(`${constant_1.IMPORT_PK_PREFIX}${core_1.KEY_SEPARATOR}`)) {
|
|
42
|
-
this.logger.debug(`Skipping other type import job in main queue handler: ${importEntity.id}`);
|
|
43
|
-
return;
|
|
44
|
-
}
|
|
45
|
-
await this.handleImport(event);
|
|
46
|
-
}
|
|
47
|
-
/**
|
|
48
|
-
* Orchestrates the processing of a single import record.
|
|
49
|
-
*/
|
|
50
|
-
async handleImport(event) {
|
|
51
|
-
const importKey = event.importEvent.importKey;
|
|
52
|
-
const importEntity = event.importEvent.importEntity;
|
|
53
|
-
const { attributes, tenantCode, type: tableName } = importEntity;
|
|
54
|
-
this.logger.debug(`Processing import job ${importKey.pk}#${importKey.sk} for table: ${tableName}`);
|
|
55
|
-
// 1. Find the correct strategies for this import's table type
|
|
56
|
-
const strategy = this.strategyMap.get(tableName);
|
|
57
|
-
if (!strategy) {
|
|
58
|
-
const error = new Error(`No import strategies registered for table: ${tableName}`);
|
|
59
|
-
this.logger.error(error);
|
|
60
|
-
await this.importService.updateStatus(importKey, import_status_enum_1.ImportStatusEnum.FAILED, { error: error.stack });
|
|
61
|
-
return;
|
|
62
|
-
}
|
|
63
26
|
try {
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
// 3. Execute all registered strategies in sequence for this record
|
|
67
|
-
await this.executeStrategy(strategy, attributes, tenantCode, importEntity);
|
|
68
|
-
// // 4. Finalize the import status as COMPLETED
|
|
69
|
-
// await this.importService.updateStatus(
|
|
70
|
-
// importKey,
|
|
71
|
-
// ImportStatusEnum.COMPLETED,
|
|
72
|
-
// { result },
|
|
73
|
-
// )
|
|
74
|
-
// this.logger.log(
|
|
75
|
-
// `Successfully completed import job: ${importKey.pk}#${importKey.sk}`,
|
|
76
|
-
// )
|
|
77
|
-
}
|
|
78
|
-
catch (error) {
|
|
79
|
-
// 5. Handle any errors during processing
|
|
80
|
-
// 5. 処理中のエラーをハンドリング
|
|
81
|
-
this.logger.error(`Failed to process import job: ${importKey.pk}#${importKey.sk}`, error);
|
|
82
|
-
await Promise.all([
|
|
83
|
-
this.importService.updateStatus(importKey, import_status_enum_1.ImportStatusEnum.FAILED, {
|
|
84
|
-
error: {
|
|
85
|
-
message: error.message,
|
|
86
|
-
stack: error.stack,
|
|
87
|
-
},
|
|
88
|
-
}),
|
|
89
|
-
this.importService.publishAlarm(event, error.stack),
|
|
90
|
-
]);
|
|
91
|
-
// 6. Increment parent job counters to track failure and trigger completion check
|
|
92
|
-
// 6. 親ジョブのカウンターを更新し、完了チェックをトリガー
|
|
93
|
-
const skParts = importEntity.sk.split(core_1.KEY_SEPARATOR);
|
|
94
|
-
const parentId = skParts.slice(0, -1).join(core_1.KEY_SEPARATOR);
|
|
95
|
-
if (parentId.startsWith(constant_1.CSV_IMPORT_PK_PREFIX)) {
|
|
96
|
-
this.logger.debug(`Updating parent job counter for FAILED child: ${importEntity.id}`);
|
|
97
|
-
const parentKey = (0, helpers_1.parseId)(parentId);
|
|
98
|
-
// Mark as failed in parent job counters
|
|
99
|
-
// 親ジョブのカウンターで失敗としてマーク
|
|
100
|
-
await this.importService.incrementParentJobCounters(parentKey, false);
|
|
27
|
+
if (event.isCsvBatch) {
|
|
28
|
+
await this.csvBatchProcessor.process(event.payload);
|
|
101
29
|
}
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
}
|
|
106
|
-
/**
|
|
107
|
-
* Executes the full lifecycle (compare, map, save) for a single strategy.
|
|
108
|
-
* @returns The result of the create/update operation or a status message.
|
|
109
|
-
*/
|
|
110
|
-
async executeStrategy(strategy, attributes, tenantCode, importEntity) {
|
|
111
|
-
// 1. Determine if there are changes
|
|
112
|
-
const compareResult = await strategy.compare(attributes, tenantCode);
|
|
113
|
-
const importKey = {
|
|
114
|
-
pk: importEntity.pk,
|
|
115
|
-
sk: importEntity.sk,
|
|
116
|
-
};
|
|
117
|
-
if (compareResult.status === comparison_status_enum_1.ComparisonStatus.EQUAL) {
|
|
118
|
-
this.logger.debug(`No changes for import job ${importEntity.id}, marking as completed.`);
|
|
119
|
-
await this.importService.updateStatus(importKey, import_status_enum_1.ImportStatusEnum.COMPLETED, { result: { status: 'EQUAL', message: 'No changes detected.' } });
|
|
120
|
-
const skParts = importEntity.sk.split(core_1.KEY_SEPARATOR);
|
|
121
|
-
const parentId = skParts.slice(0, -1).join(core_1.KEY_SEPARATOR);
|
|
122
|
-
if (parentId.startsWith(constant_1.CSV_IMPORT_PK_PREFIX)) {
|
|
123
|
-
this.logger.debug(`Updating parent job counter for EQUAL status child: ${importEntity.id}`);
|
|
124
|
-
const parentKey = (0, helpers_1.parseId)(parentId);
|
|
125
|
-
// Since the status is EQUAL, the child "succeeded" in its processing.
|
|
126
|
-
await this.importService.incrementParentJobCounters(parentKey, true);
|
|
30
|
+
else {
|
|
31
|
+
// Fallback to single import (or add more if/else for ZIP)
|
|
32
|
+
await this.singleImportProcessor.process(event.payload);
|
|
127
33
|
}
|
|
128
|
-
return; // Stop execution for this case
|
|
129
34
|
}
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
const commandService = strategy.getCommandService();
|
|
134
|
-
const { type: tableName } = importEntity;
|
|
135
|
-
const publishMode = this.publishModeMap.get(tableName) ?? constant_1.ImportPublishMode.ASYNC;
|
|
136
|
-
let result;
|
|
137
|
-
// 3. Execute the appropriate command
|
|
138
|
-
const invokeContext = (0, core_1.extractInvokeContext)();
|
|
139
|
-
const options = {
|
|
140
|
-
invokeContext,
|
|
141
|
-
source: importEntity.id,
|
|
142
|
-
};
|
|
143
|
-
if (compareResult.status === comparison_status_enum_1.ComparisonStatus.NOT_EXIST) {
|
|
144
|
-
result =
|
|
145
|
-
publishMode === constant_1.ImportPublishMode.SYNC
|
|
146
|
-
? await commandService.publishSync(mappedData, options)
|
|
147
|
-
: await commandService.publishAsync(mappedData, options);
|
|
148
|
-
// 4. Finalize the import status as COMPLETED
|
|
149
|
-
await this.importService.updateStatus(importKey, import_status_enum_1.ImportStatusEnum.PROCESSING, { result: result });
|
|
150
|
-
this.logger.debug(`Successfully completed import job: ${importKey.pk}#${importKey.sk} with publish mode: ${publishMode}`);
|
|
151
|
-
}
|
|
152
|
-
else {
|
|
153
|
-
result =
|
|
154
|
-
publishMode === constant_1.ImportPublishMode.SYNC
|
|
155
|
-
? await commandService.publishPartialUpdateSync(mappedData, options)
|
|
156
|
-
: await commandService.publishPartialUpdateAsync(mappedData, options);
|
|
157
|
-
await this.importService.updateStatus(importKey, import_status_enum_1.ImportStatusEnum.PROCESSING, { result: result });
|
|
158
|
-
this.logger.debug(`Successfully completed import job: ${importKey.pk}#${importKey.sk} with publish mode: ${publishMode}`);
|
|
35
|
+
catch (error) {
|
|
36
|
+
this.logger.error('Failed to process message in ImportQueueEventHandler', event, error);
|
|
37
|
+
throw error; // Ensure SQS retries
|
|
159
38
|
}
|
|
160
39
|
}
|
|
161
40
|
};
|
|
162
41
|
exports.ImportQueueEventHandler = ImportQueueEventHandler;
|
|
163
42
|
exports.ImportQueueEventHandler = ImportQueueEventHandler = ImportQueueEventHandler_1 = __decorate([
|
|
164
43
|
(0, core_1.EventHandler)(import_queue_event_1.ImportQueueEvent),
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
__metadata("design:paramtypes", [import_service_1.ImportService,
|
|
168
|
-
Map,
|
|
169
|
-
Map])
|
|
44
|
+
__metadata("design:paramtypes", [csv_batch_processor_1.CsvBatchProcessor,
|
|
45
|
+
single_import_processor_1.SingleImportProcessor])
|
|
170
46
|
], ImportQueueEventHandler);
|
|
171
47
|
//# sourceMappingURL=import.queue.event.handler.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"import.queue.event.handler.js","sourceRoot":"","sources":["../../src/event/import.queue.event.handler.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"import.queue.event.handler.js","sourceRoot":"","sources":["../../src/event/import.queue.event.handler.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,oDAAuE;AACvE,2CAAuC;AAEvC,6DAAuD;AACvD,yEAAmE;AACnE,iFAA2E;AAGpE,IAAM,uBAAuB,+BAA7B,MAAM,uBAAuB;IAKlC,YACmB,iBAAoC,EACpC,qBAA4C;QAD5C,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,0BAAqB,GAArB,qBAAqB,CAAuB;QAJ9C,WAAM,GAAG,IAAI,eAAM,CAAC,yBAAuB,CAAC,IAAI,CAAC,CAAA;IAK/D,CAAC;IAEJ,KAAK,CAAC,OAAO,CAAC,KAAuB;QACnC,IAAI,CAAC;YACH,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;gBACrB,MAAM,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;YACrD,CAAC;iBAAM,CAAC;gBACN,0DAA0D;gBAC1D,MAAM,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;YACzD,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,sDAAsD,EACtD,KAAK,EACL,KAAK,CACN,CAAA;YACD,MAAM,KAAK,CAAA,CAAC,qBAAqB;QACnC,CAAC;IACH,CAAC;CACF,CAAA;AA3BY,0DAAuB;kCAAvB,uBAAuB;IADnC,IAAA,mBAAY,EAAC,qCAAgB,CAAC;qCAOS,uCAAiB;QACb,+CAAqB;GAPpD,uBAAuB,CA2BnC"}
|
|
@@ -1,17 +1,38 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.ImportQueueEvent = void 0;
|
|
4
|
+
const sqs_constant_1 = require("../constant/sqs.constant");
|
|
4
5
|
const import_event_1 = require("./import.event");
|
|
5
6
|
class ImportQueueEvent {
|
|
6
7
|
fromSqsRecord(record) {
|
|
7
|
-
Object.assign(this, record, {
|
|
8
|
-
source: record.eventSourceARN,
|
|
9
|
-
});
|
|
8
|
+
Object.assign(this, record, { source: record.eventSourceARN });
|
|
10
9
|
return this;
|
|
11
10
|
}
|
|
11
|
+
get payload() {
|
|
12
|
+
if (!this._payload) {
|
|
13
|
+
let parsed = JSON.parse(this.body);
|
|
14
|
+
if (parsed.Type === 'Notification' && parsed.Message) {
|
|
15
|
+
parsed = JSON.parse(parsed.Message);
|
|
16
|
+
}
|
|
17
|
+
this._payload = parsed;
|
|
18
|
+
}
|
|
19
|
+
return this._payload;
|
|
20
|
+
}
|
|
21
|
+
get isCsvBatch() {
|
|
22
|
+
return this.payload?.action === sqs_constant_1.ACTION_CSV_BATCH_PROCESS;
|
|
23
|
+
}
|
|
24
|
+
// ----------------------------------------------------------------
|
|
25
|
+
// BACKWARD COMPATIBILITY: Fixes TS2339 & TypeError
|
|
26
|
+
// ----------------------------------------------------------------
|
|
12
27
|
get importEvent() {
|
|
13
28
|
if (!this._importEvent) {
|
|
14
|
-
|
|
29
|
+
// 1. Create a shallow copy of the payload
|
|
30
|
+
const safePayload = { ...this.payload };
|
|
31
|
+
// 2. Delete properties that conflict with ImportEvent's read-only getters
|
|
32
|
+
delete safePayload.tableName;
|
|
33
|
+
delete safePayload.action; // (Optional) clean up the action tag
|
|
34
|
+
// 3. Safely instantiate
|
|
35
|
+
this._importEvent = new import_event_1.ImportEvent(safePayload);
|
|
15
36
|
}
|
|
16
37
|
return this._importEvent;
|
|
17
38
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"import.queue.event.js","sourceRoot":"","sources":["../../src/event/import.queue.event.ts"],"names":[],"mappings":";;;AAOA,iDAA4C;AAE5C,MAAa,gBAAgB;
|
|
1
|
+
{"version":3,"file":"import.queue.event.js","sourceRoot":"","sources":["../../src/event/import.queue.event.ts"],"names":[],"mappings":";;;AAOA,2DAAmE;AACnE,iDAA4C;AAE5C,MAAa,gBAAgB;IAe3B,aAAa,CAAC,MAAiB;QAC7B,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC,CAAA;QAC9D,OAAO,IAAI,CAAA;IACb,CAAC;IAED,IAAI,OAAO;QACT,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,IAAI,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAClC,IAAI,MAAM,CAAC,IAAI,KAAK,cAAc,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACrD,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;YACrC,CAAC;YACD,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAA;QACxB,CAAC;QACD,OAAO,IAAI,CAAC,QAAQ,CAAA;IACtB,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,OAAO,EAAE,MAAM,KAAK,uCAAwB,CAAA;IAC1D,CAAC;IAED,mEAAmE;IACnE,mDAAmD;IACnD,mEAAmE;IACnE,IAAI,WAAW;QACb,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,0CAA0C;YAC1C,MAAM,WAAW,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,CAAA;YAEvC,0EAA0E;YAC1E,OAAO,WAAW,CAAC,SAAS,CAAA;YAC5B,OAAO,WAAW,CAAC,MAAM,CAAA,CAAC,qCAAqC;YAE/D,wBAAwB;YACxB,IAAI,CAAC,YAAY,GAAG,IAAI,0BAAW,CAAC,WAAW,CAAC,CAAA;QAClD,CAAC;QACD,OAAO,IAAI,CAAC,YAAY,CAAA;IAC1B,CAAC;CACF;AApDD,4CAoDC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { ImportPublishMode } from '../../constant';
|
|
2
|
+
import { SqsBatchPayload } from '../../constant/sqs.constant';
|
|
3
|
+
import { IProcessStrategy } from '../../interface/processing-strategy.interface';
|
|
4
|
+
export declare class CsvBatchProcessor {
|
|
5
|
+
private readonly strategyMap;
|
|
6
|
+
private readonly publishModeMap;
|
|
7
|
+
private readonly logger;
|
|
8
|
+
constructor(strategyMap: Map<string, IProcessStrategy<any, any>>, publishModeMap: Map<string, ImportPublishMode>);
|
|
9
|
+
process(payload: SqsBatchPayload): Promise<void>;
|
|
10
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
12
|
+
return function (target, key) { decorator(target, key, paramIndex); }
|
|
13
|
+
};
|
|
14
|
+
var CsvBatchProcessor_1;
|
|
15
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
+
exports.CsvBatchProcessor = void 0;
|
|
17
|
+
const core_1 = require("@mbc-cqrs-serverless/core");
|
|
18
|
+
const common_1 = require("@nestjs/common");
|
|
19
|
+
const constant_1 = require("../../constant");
|
|
20
|
+
const comparison_status_enum_1 = require("../../enum/comparison-status.enum");
|
|
21
|
+
const import_module_definition_1 = require("../../import.module-definition");
|
|
22
|
+
let CsvBatchProcessor = CsvBatchProcessor_1 = class CsvBatchProcessor {
|
|
23
|
+
constructor(strategyMap, publishModeMap) {
|
|
24
|
+
this.strategyMap = strategyMap;
|
|
25
|
+
this.publishModeMap = publishModeMap;
|
|
26
|
+
this.logger = new common_1.Logger(CsvBatchProcessor_1.name);
|
|
27
|
+
}
|
|
28
|
+
async process(payload) {
|
|
29
|
+
this.logger.debug(`Processing CSV Batch for table: ${payload.tableName}`);
|
|
30
|
+
const { tableName, tenantCode, sourceId, s3Key, items } = payload;
|
|
31
|
+
const strategy = this.strategyMap.get(tableName);
|
|
32
|
+
if (!strategy)
|
|
33
|
+
throw new Error(`No process strategy registered for table: ${tableName}`);
|
|
34
|
+
const publishMode = this.publishModeMap.get(tableName) ?? constant_1.ImportPublishMode.ASYNC;
|
|
35
|
+
const commandService = strategy.getCommandService();
|
|
36
|
+
const options = {
|
|
37
|
+
invokeContext: (0, core_1.extractInvokeContext)(),
|
|
38
|
+
source: sourceId,
|
|
39
|
+
};
|
|
40
|
+
for (const data of items) {
|
|
41
|
+
const compareResult = await strategy.compare({ ...data, __s3Key: s3Key }, tenantCode);
|
|
42
|
+
if (compareResult.status === comparison_status_enum_1.ComparisonStatus.EQUAL)
|
|
43
|
+
continue;
|
|
44
|
+
const mappedData = await strategy.map(compareResult.status, data, tenantCode, compareResult.existingData);
|
|
45
|
+
if (compareResult.status === comparison_status_enum_1.ComparisonStatus.NOT_EXIST) {
|
|
46
|
+
publishMode === constant_1.ImportPublishMode.SYNC
|
|
47
|
+
? await commandService.publishSync(mappedData, options)
|
|
48
|
+
: await commandService.publishAsync(mappedData, options);
|
|
49
|
+
}
|
|
50
|
+
else {
|
|
51
|
+
publishMode === constant_1.ImportPublishMode.SYNC
|
|
52
|
+
? await commandService.publishPartialUpdateSync(mappedData, options)
|
|
53
|
+
: await commandService.publishPartialUpdateAsync(mappedData, options);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
};
|
|
58
|
+
exports.CsvBatchProcessor = CsvBatchProcessor;
|
|
59
|
+
exports.CsvBatchProcessor = CsvBatchProcessor = CsvBatchProcessor_1 = __decorate([
|
|
60
|
+
(0, common_1.Injectable)(),
|
|
61
|
+
__param(0, (0, common_1.Inject)(import_module_definition_1.PROCESS_STRATEGY_MAP)),
|
|
62
|
+
__param(1, (0, common_1.Inject)(import_module_definition_1.PUBLISH_MODE_MAP)),
|
|
63
|
+
__metadata("design:paramtypes", [Map,
|
|
64
|
+
Map])
|
|
65
|
+
], CsvBatchProcessor);
|
|
66
|
+
//# sourceMappingURL=csv-batch.processor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"csv-batch.processor.js","sourceRoot":"","sources":["../../../src/event/processor/csv-batch.processor.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,oDAKkC;AAClC,2CAA2D;AAE3D,6CAAkD;AAElD,8EAAoE;AACpE,6EAGuC;AAIhC,IAAM,iBAAiB,yBAAvB,MAAM,iBAAiB;IAG5B,YAEE,WAAqE,EAErE,cAA+D;QAF9C,gBAAW,GAAX,WAAW,CAAyC;QAEpD,mBAAc,GAAd,cAAc,CAAgC;QANhD,WAAM,GAAG,IAAI,eAAM,CAAC,mBAAiB,CAAC,IAAI,CAAC,CAAA;IAOzD,CAAC;IAEJ,KAAK,CAAC,OAAO,CAAC,OAAwB;QACpC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAA;QACzE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,OAAO,CAAA;QAEjE,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAChD,IAAI,CAAC,QAAQ;YACX,MAAM,IAAI,KAAK,CAAC,6CAA6C,SAAS,EAAE,CAAC,CAAA;QAE3E,MAAM,WAAW,GACf,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,4BAAiB,CAAC,KAAK,CAAA;QAC/D,MAAM,cAAc,GAAG,QAAQ,CAAC,iBAAiB,EAAE,CAAA;QACnD,MAAM,OAAO,GAAoB;YAC/B,aAAa,EAAE,IAAA,2BAAoB,GAAE;YACrC,MAAM,EAAE,QAAQ;SACjB,CAAA;QAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,aAAa,GAAG,MAAM,QAAQ,CAAC,OAAO,CAC1C,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAC3B,UAAU,CACX,CAAA;YACD,IAAI,aAAa,CAAC,MAAM,KAAK,yCAAgB,CAAC,KAAK;gBAAE,SAAQ;YAE7D,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,GAAG,CACnC,aAAa,CAAC,MAAM,EACpB,IAAI,EACJ,UAAU,EACV,aAAa,CAAC,YAAY,CAC3B,CAAA;YAED,IAAI,aAAa,CAAC,MAAM,KAAK,yCAAgB,CAAC,SAAS,EAAE,CAAC;gBACxD,WAAW,KAAK,4BAAiB,CAAC,IAAI;oBACpC,CAAC,CAAC,MAAM,cAAc,CAAC,WAAW,CAC9B,UAA+B,EAC/B,OAAO,CACR;oBACH,CAAC,CAAC,MAAM,cAAc,CAAC,YAAY,CAC/B,UAA+B,EAC/B,OAAO,CACR,CAAA;YACP,CAAC;iBAAM,CAAC;gBACN,WAAW,KAAK,4BAAiB,CAAC,IAAI;oBACpC,CAAC,CAAC,MAAM,cAAc,CAAC,wBAAwB,CAC3C,UAAsC,EACtC,OAAO,CACR;oBACH,CAAC,CAAC,MAAM,cAAc,CAAC,yBAAyB,CAC5C,UAAsC,EACtC,OAAO,CACR,CAAA;YACP,CAAC;QACH,CAAC;IACH,CAAC;CACF,CAAA;AA/DY,8CAAiB;4BAAjB,iBAAiB;IAD7B,IAAA,mBAAU,GAAE;IAKR,WAAA,IAAA,eAAM,EAAC,+CAAoB,CAAC,CAAA;IAE5B,WAAA,IAAA,eAAM,EAAC,2CAAgB,CAAC,CAAA;qCADK,GAAG;QAEA,GAAG;GAP3B,iBAAiB,CA+D7B"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { ImportPublishMode } from '../../constant';
|
|
2
|
+
import { ImportService } from '../../import.service';
|
|
3
|
+
import { IProcessStrategy } from '../../interface/processing-strategy.interface';
|
|
4
|
+
export declare class SingleImportProcessor {
|
|
5
|
+
private readonly importService;
|
|
6
|
+
private readonly strategyMap;
|
|
7
|
+
private readonly publishModeMap;
|
|
8
|
+
private readonly logger;
|
|
9
|
+
constructor(importService: ImportService, strategyMap: Map<string, IProcessStrategy<any, any>>, publishModeMap: Map<string, ImportPublishMode>);
|
|
10
|
+
process(payload: any): Promise<void>;
|
|
11
|
+
private executeStrategy;
|
|
12
|
+
}
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
12
|
+
return function (target, key) { decorator(target, key, paramIndex); }
|
|
13
|
+
};
|
|
14
|
+
var SingleImportProcessor_1;
|
|
15
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
+
exports.SingleImportProcessor = void 0;
|
|
17
|
+
const core_1 = require("@mbc-cqrs-serverless/core");
|
|
18
|
+
const common_1 = require("@nestjs/common");
|
|
19
|
+
const constant_1 = require("../../constant");
|
|
20
|
+
const comparison_status_enum_1 = require("../../enum/comparison-status.enum");
|
|
21
|
+
const import_status_enum_1 = require("../../enum/import-status.enum");
|
|
22
|
+
const helpers_1 = require("../../helpers");
|
|
23
|
+
const import_module_definition_1 = require("../../import.module-definition");
|
|
24
|
+
const import_service_1 = require("../../import.service");
|
|
25
|
+
let SingleImportProcessor = SingleImportProcessor_1 = class SingleImportProcessor {
|
|
26
|
+
constructor(importService, strategyMap, publishModeMap) {
|
|
27
|
+
this.importService = importService;
|
|
28
|
+
this.strategyMap = strategyMap;
|
|
29
|
+
this.publishModeMap = publishModeMap;
|
|
30
|
+
this.logger = new common_1.Logger(SingleImportProcessor_1.name);
|
|
31
|
+
}
|
|
32
|
+
async process(payload) {
|
|
33
|
+
const importEntity = payload.importEntity;
|
|
34
|
+
// Guard clause: Ensure this is actually an import job we care about
|
|
35
|
+
if (!importEntity?.id?.startsWith(`${constant_1.IMPORT_PK_PREFIX}${core_1.KEY_SEPARATOR}`)) {
|
|
36
|
+
this.logger.debug(`Skipping other type import job: ${importEntity?.id}`);
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
const importKey = payload.importKey;
|
|
40
|
+
const { attributes, tenantCode, type: tableName } = importEntity;
|
|
41
|
+
this.logger.debug(`Processing single import job ${importKey.pk}#${importKey.sk} for table: ${tableName}`);
|
|
42
|
+
const strategy = this.strategyMap.get(tableName);
|
|
43
|
+
if (!strategy) {
|
|
44
|
+
const error = new Error(`No import strategies registered for table: ${tableName}`);
|
|
45
|
+
this.logger.error(error);
|
|
46
|
+
await this.importService.updateStatus(importKey, import_status_enum_1.ImportStatusEnum.FAILED, {
|
|
47
|
+
error: error.stack,
|
|
48
|
+
});
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
try {
|
|
52
|
+
await this.importService.updateStatus(importKey, import_status_enum_1.ImportStatusEnum.PROCESSING);
|
|
53
|
+
await this.executeStrategy(strategy, attributes, tenantCode, importEntity);
|
|
54
|
+
}
|
|
55
|
+
catch (error) {
|
|
56
|
+
this.logger.error(`Failed to process import job: ${importKey.pk}#${importKey.sk}`, error);
|
|
57
|
+
await Promise.all([
|
|
58
|
+
this.importService.updateStatus(importKey, import_status_enum_1.ImportStatusEnum.FAILED, {
|
|
59
|
+
error: {
|
|
60
|
+
message: error.message,
|
|
61
|
+
stack: error.stack,
|
|
62
|
+
},
|
|
63
|
+
}),
|
|
64
|
+
this.importService.publishAlarm(payload, error.stack),
|
|
65
|
+
]);
|
|
66
|
+
// If this single import was part of a larger parent job, inform the parent it failed
|
|
67
|
+
const skParts = importEntity.sk.split(core_1.KEY_SEPARATOR);
|
|
68
|
+
const parentId = skParts.slice(0, -1).join(core_1.KEY_SEPARATOR);
|
|
69
|
+
if (parentId.startsWith(constant_1.CSV_IMPORT_PK_PREFIX)) {
|
|
70
|
+
const parentKey = (0, helpers_1.parseId)(parentId);
|
|
71
|
+
await this.importService.incrementParentJobCounters(parentKey, false);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
async executeStrategy(strategy, attributes, tenantCode, importEntity) {
|
|
76
|
+
const compareResult = await strategy.compare(attributes, tenantCode);
|
|
77
|
+
const importKey = { pk: importEntity.pk, sk: importEntity.sk };
|
|
78
|
+
// SCENARIO 1: Data is identical, no update needed
|
|
79
|
+
if (compareResult.status === comparison_status_enum_1.ComparisonStatus.EQUAL) {
|
|
80
|
+
this.logger.debug(`No changes for import job ${importEntity.id}, marking as completed.`);
|
|
81
|
+
await this.importService.updateStatus(importKey, import_status_enum_1.ImportStatusEnum.COMPLETED, {
|
|
82
|
+
result: { status: 'EQUAL', message: 'No changes detected.' },
|
|
83
|
+
});
|
|
84
|
+
const skParts = importEntity.sk.split(core_1.KEY_SEPARATOR);
|
|
85
|
+
const parentId = skParts.slice(0, -1).join(core_1.KEY_SEPARATOR);
|
|
86
|
+
if (parentId.startsWith(constant_1.CSV_IMPORT_PK_PREFIX)) {
|
|
87
|
+
const parentKey = (0, helpers_1.parseId)(parentId);
|
|
88
|
+
await this.importService.incrementParentJobCounters(parentKey, true);
|
|
89
|
+
}
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
// SCENARIO 2: Data is new or changed, requires mapping and publishing
|
|
93
|
+
const mappedData = await strategy.map(compareResult.status, attributes, tenantCode, compareResult.existingData);
|
|
94
|
+
const commandService = strategy.getCommandService();
|
|
95
|
+
const { type: tableName } = importEntity;
|
|
96
|
+
const publishMode = this.publishModeMap.get(tableName) ?? constant_1.ImportPublishMode.ASYNC;
|
|
97
|
+
const invokeContext = (0, core_1.extractInvokeContext)();
|
|
98
|
+
const options = { invokeContext, source: importEntity.id };
|
|
99
|
+
let result;
|
|
100
|
+
if (compareResult.status === comparison_status_enum_1.ComparisonStatus.NOT_EXIST) {
|
|
101
|
+
// Create new record
|
|
102
|
+
result =
|
|
103
|
+
publishMode === constant_1.ImportPublishMode.SYNC
|
|
104
|
+
? await commandService.publishSync(mappedData, options)
|
|
105
|
+
: await commandService.publishAsync(mappedData, options);
|
|
106
|
+
}
|
|
107
|
+
else {
|
|
108
|
+
// Update existing record
|
|
109
|
+
result =
|
|
110
|
+
publishMode === constant_1.ImportPublishMode.SYNC
|
|
111
|
+
? await commandService.publishPartialUpdateSync(mappedData, options)
|
|
112
|
+
: await commandService.publishPartialUpdateAsync(mappedData, options);
|
|
113
|
+
}
|
|
114
|
+
await this.importService.updateStatus(importKey, import_status_enum_1.ImportStatusEnum.COMPLETED, { result });
|
|
115
|
+
const skParts = importEntity.sk.split(core_1.KEY_SEPARATOR);
|
|
116
|
+
const parentId = skParts.slice(0, -1).join(core_1.KEY_SEPARATOR);
|
|
117
|
+
if (parentId.startsWith(constant_1.CSV_IMPORT_PK_PREFIX)) {
|
|
118
|
+
const parentKey = (0, helpers_1.parseId)(parentId);
|
|
119
|
+
await this.importService.incrementParentJobCounters(parentKey, true);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
};
|
|
123
|
+
exports.SingleImportProcessor = SingleImportProcessor;
|
|
124
|
+
exports.SingleImportProcessor = SingleImportProcessor = SingleImportProcessor_1 = __decorate([
|
|
125
|
+
(0, common_1.Injectable)(),
|
|
126
|
+
__param(1, (0, common_1.Inject)(import_module_definition_1.PROCESS_STRATEGY_MAP)),
|
|
127
|
+
__param(2, (0, common_1.Inject)(import_module_definition_1.PUBLISH_MODE_MAP)),
|
|
128
|
+
__metadata("design:paramtypes", [import_service_1.ImportService,
|
|
129
|
+
Map,
|
|
130
|
+
Map])
|
|
131
|
+
], SingleImportProcessor);
|
|
132
|
+
//# sourceMappingURL=single-import.processor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"single-import.processor.js","sourceRoot":"","sources":["../../../src/event/processor/single-import.processor.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,oDAMkC;AAClC,2CAA2D;AAE3D,6CAIuB;AAEvB,8EAAoE;AACpE,sEAAgE;AAChE,2CAAuC;AACvC,6EAGuC;AACvC,yDAAoD;AAI7C,IAAM,qBAAqB,6BAA3B,MAAM,qBAAqB;IAGhC,YACmB,aAA4B,EAE7C,WAAqE,EAErE,cAA+D;QAJ9C,kBAAa,GAAb,aAAa,CAAe;QAE5B,gBAAW,GAAX,WAAW,CAAyC;QAEpD,mBAAc,GAAd,cAAc,CAAgC;QAPhD,WAAM,GAAG,IAAI,eAAM,CAAC,uBAAqB,CAAC,IAAI,CAAC,CAAA;IAQ7D,CAAC;IAEJ,KAAK,CAAC,OAAO,CAAC,OAAY;QACxB,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,CAAA;QAEzC,oEAAoE;QACpE,IAAI,CAAC,YAAY,EAAE,EAAE,EAAE,UAAU,CAAC,GAAG,2BAAgB,GAAG,oBAAa,EAAE,CAAC,EAAE,CAAC;YACzE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,YAAY,EAAE,EAAE,EAAE,CAAC,CAAA;YACxE,OAAM;QACR,CAAC;QAED,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAA;QACnC,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,YAAY,CAAA;QAEhE,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,gCAAgC,SAAS,CAAC,EAAE,IAAI,SAAS,CAAC,EAAE,eAAe,SAAS,EAAE,CACvF,CAAA;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAChD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,KAAK,GAAG,IAAI,KAAK,CACrB,8CAA8C,SAAS,EAAE,CAC1D,CAAA;YACD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;YACxB,MAAM,IAAI,CAAC,aAAa,CAAC,YAAY,CACnC,SAAS,EACT,qCAAgB,CAAC,MAAM,EACvB;gBACE,KAAK,EAAE,KAAK,CAAC,KAAK;aACnB,CACF,CAAA;YACD,OAAM;QACR,CAAC;QAED,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,aAAa,CAAC,YAAY,CACnC,SAAS,EACT,qCAAgB,CAAC,UAAU,CAC5B,CAAA;YACD,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,YAAY,CAAC,CAAA;QAC5E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,iCAAiC,SAAS,CAAC,EAAE,IAAI,SAAS,CAAC,EAAE,EAAE,EAC/D,KAAK,CACN,CAAA;YAED,MAAM,OAAO,CAAC,GAAG,CAAC;gBAChB,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,SAAS,EAAE,qCAAgB,CAAC,MAAM,EAAE;oBAClE,KAAK,EAAE;wBACL,OAAO,EAAG,KAAe,CAAC,OAAO;wBACjC,KAAK,EAAG,KAAe,CAAC,KAAK;qBAC9B;iBACF,CAAC;gBACF,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,OAAO,EAAG,KAAe,CAAC,KAAK,CAAC;aACjE,CAAC,CAAA;YAEF,qFAAqF;YACrF,MAAM,OAAO,GAAG,YAAY,CAAC,EAAE,CAAC,KAAK,CAAC,oBAAa,CAAC,CAAA;YACpD,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,oBAAa,CAAC,CAAA;YAEzD,IAAI,QAAQ,CAAC,UAAU,CAAC,+BAAoB,CAAC,EAAE,CAAC;gBAC9C,MAAM,SAAS,GAAG,IAAA,iBAAO,EAAC,QAAQ,CAAC,CAAA;gBACnC,MAAM,IAAI,CAAC,aAAa,CAAC,0BAA0B,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;YACvE,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,QAAoC,EACpC,UAA+B,EAC/B,UAAkB,EAClB,YAA0B;QAE1B,MAAM,aAAa,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,UAAU,CAAC,CAAA;QACpE,MAAM,SAAS,GAAG,EAAE,EAAE,EAAE,YAAY,CAAC,EAAE,EAAE,EAAE,EAAE,YAAY,CAAC,EAAE,EAAE,CAAA;QAE9D,kDAAkD;QAClD,IAAI,aAAa,CAAC,MAAM,KAAK,yCAAgB,CAAC,KAAK,EAAE,CAAC;YACpD,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,6BAA6B,YAAY,CAAC,EAAE,yBAAyB,CACtE,CAAA;YACD,MAAM,IAAI,CAAC,aAAa,CAAC,YAAY,CACnC,SAAS,EACT,qCAAgB,CAAC,SAAS,EAC1B;gBACE,MAAM,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,sBAAsB,EAAE;aAC7D,CACF,CAAA;YAED,MAAM,OAAO,GAAG,YAAY,CAAC,EAAE,CAAC,KAAK,CAAC,oBAAa,CAAC,CAAA;YACpD,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,oBAAa,CAAC,CAAA;YAEzD,IAAI,QAAQ,CAAC,UAAU,CAAC,+BAAoB,CAAC,EAAE,CAAC;gBAC9C,MAAM,SAAS,GAAG,IAAA,iBAAO,EAAC,QAAQ,CAAC,CAAA;gBACnC,MAAM,IAAI,CAAC,aAAa,CAAC,0BAA0B,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;YACtE,CAAC;YACD,OAAM;QACR,CAAC;QAED,sEAAsE;QACtE,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,GAAG,CACnC,aAAa,CAAC,MAAM,EACpB,UAAU,EACV,UAAU,EACV,aAAa,CAAC,YAAY,CAC3B,CAAA;QAED,MAAM,cAAc,GAAG,QAAQ,CAAC,iBAAiB,EAAE,CAAA;QACnD,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,YAAY,CAAA;QACxC,MAAM,WAAW,GACf,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,4BAAiB,CAAC,KAAK,CAAA;QAE/D,MAAM,aAAa,GAAG,IAAA,2BAAoB,GAAE,CAAA;QAC5C,MAAM,OAAO,GAAoB,EAAE,aAAa,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,EAAE,CAAA;QAE3E,IAAI,MAAW,CAAA;QAEf,IAAI,aAAa,CAAC,MAAM,KAAK,yCAAgB,CAAC,SAAS,EAAE,CAAC;YACxD,oBAAoB;YACpB,MAAM;gBACJ,WAAW,KAAK,4BAAiB,CAAC,IAAI;oBACpC,CAAC,CAAC,MAAM,cAAc,CAAC,WAAW,CAC9B,UAA+B,EAC/B,OAAO,CACR;oBACH,CAAC,CAAC,MAAM,cAAc,CAAC,YAAY,CAC/B,UAA+B,EAC/B,OAAO,CACR,CAAA;QACT,CAAC;aAAM,CAAC;YACN,yBAAyB;YACzB,MAAM;gBACJ,WAAW,KAAK,4BAAiB,CAAC,IAAI;oBACpC,CAAC,CAAC,MAAM,cAAc,CAAC,wBAAwB,CAC3C,UAAsC,EACtC,OAAO,CACR;oBACH,CAAC,CAAC,MAAM,cAAc,CAAC,yBAAyB,CAC5C,UAAsC,EACtC,OAAO,CACR,CAAA;QACT,CAAC;QAED,MAAM,IAAI,CAAC,aAAa,CAAC,YAAY,CACnC,SAAS,EACT,qCAAgB,CAAC,SAAS,EAC1B,EAAE,MAAM,EAAE,CACX,CAAA;QAED,MAAM,OAAO,GAAG,YAAY,CAAC,EAAE,CAAC,KAAK,CAAC,oBAAa,CAAC,CAAA;QACpD,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,oBAAa,CAAC,CAAA;QAEzD,IAAI,QAAQ,CAAC,UAAU,CAAC,+BAAoB,CAAC,EAAE,CAAC;YAC9C,MAAM,SAAS,GAAG,IAAA,iBAAO,EAAC,QAAQ,CAAC,CAAA;YACnC,MAAM,IAAI,CAAC,aAAa,CAAC,0BAA0B,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;QACtE,CAAC;IACH,CAAC;CACF,CAAA;AAtKY,sDAAqB;gCAArB,qBAAqB;IADjC,IAAA,mBAAU,GAAE;IAMR,WAAA,IAAA,eAAM,EAAC,+CAAoB,CAAC,CAAA;IAE5B,WAAA,IAAA,eAAM,EAAC,2CAAgB,CAAC,CAAA;qCAHO,8BAAa;QAEf,GAAG;QAEA,GAAG;GAR3B,qBAAqB,CAsKjC"}
|
|
@@ -65,8 +65,15 @@ let ZipImportQueueEventHandler = ZipImportQueueEventHandler_1 = class ZipImportQ
|
|
|
65
65
|
this.zipOrchestratorArn = this.configService.get('SFN_IMPORT_ZIP_ORCHESTRATOR_ARN');
|
|
66
66
|
}
|
|
67
67
|
async execute(event) {
|
|
68
|
-
|
|
69
|
-
|
|
68
|
+
if (event.isCsvBatch) {
|
|
69
|
+
this.logger.debug('Skipping batch processing payload in ZipImportQueueEventHandler');
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
const importEntity = event.importEvent?.importEntity;
|
|
73
|
+
if (!importEntity)
|
|
74
|
+
return;
|
|
75
|
+
if (!importEntity.id.startsWith(`${constant_1.ZIP_IMPORT_PK_PREFIX}${core_1.KEY_SEPARATOR}`)) {
|
|
76
|
+
this.logger.debug(`Skipping other type import job in zip queue handler: ${importEntity.id}`);
|
|
70
77
|
return;
|
|
71
78
|
}
|
|
72
79
|
const importKey = event.importEvent.importKey;
|