@webiny/api-sync-system 6.3.0 → 6.4.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/constants.js +2 -1
- package/constants.js.map +1 -1
- package/index.js +0 -2
- package/package.json +14 -14
- package/resolver/app/RecordHandler.js +83 -105
- package/resolver/app/RecordHandler.js.map +1 -1
- package/resolver/app/RecordsValidation.js +11 -14
- package/resolver/app/RecordsValidation.js.map +1 -1
- package/resolver/app/ResolverApplication.js +24 -31
- package/resolver/app/ResolverApplication.js.map +1 -1
- package/resolver/app/abstractions/RecordHandler.js +0 -3
- package/resolver/app/abstractions/RecordsValidation.js +0 -3
- package/resolver/app/abstractions/ResolverApplication.js +0 -3
- package/resolver/app/abstractions/ResolverRecord.js +0 -3
- package/resolver/app/bundler/BaseBundle.js +17 -16
- package/resolver/app/bundler/BaseBundle.js.map +1 -1
- package/resolver/app/bundler/Bundler.js +13 -18
- package/resolver/app/bundler/Bundler.js.map +1 -1
- package/resolver/app/bundler/Bundles.js +21 -26
- package/resolver/app/bundler/Bundles.js.map +1 -1
- package/resolver/app/bundler/CommandBundle.js +7 -16
- package/resolver/app/bundler/CommandBundle.js.map +1 -1
- package/resolver/app/bundler/TableBundle.js +6 -13
- package/resolver/app/bundler/TableBundle.js.map +1 -1
- package/resolver/app/bundler/types.js +0 -3
- package/resolver/app/commandHandler/DeleteCommandHandler.js +22 -29
- package/resolver/app/commandHandler/DeleteCommandHandler.js.map +1 -1
- package/resolver/app/commandHandler/PutCommandHandler.js +16 -22
- package/resolver/app/commandHandler/PutCommandHandler.js.map +1 -1
- package/resolver/app/commandHandler/delete.js +11 -12
- package/resolver/app/commandHandler/delete.js.map +1 -1
- package/resolver/app/commandHandler/put.js +11 -12
- package/resolver/app/commandHandler/put.js.map +1 -1
- package/resolver/app/data/SourceDataContainer.js +28 -30
- package/resolver/app/data/SourceDataContainer.js.map +1 -1
- package/resolver/app/data/types.js +0 -3
- package/resolver/app/fetcher/Fetcher.js +84 -109
- package/resolver/app/fetcher/Fetcher.js.map +1 -1
- package/resolver/app/fetcher/types.js +0 -3
- package/resolver/app/ingestor/Ingestor.js +27 -32
- package/resolver/app/ingestor/Ingestor.js.map +1 -1
- package/resolver/app/ingestor/IngestorResult.js +28 -32
- package/resolver/app/ingestor/IngestorResult.js.map +1 -1
- package/resolver/app/ingestor/types.js +0 -3
- package/resolver/app/storer/Storer.js +71 -85
- package/resolver/app/storer/Storer.js.map +1 -1
- package/resolver/app/storer/types.js +0 -3
- package/resolver/app/transform/TransformHandler.js +45 -56
- package/resolver/app/transform/TransformHandler.js.map +1 -1
- package/resolver/app/transform/middleware.js +13 -20
- package/resolver/app/transform/middleware.js.map +1 -1
- package/resolver/app/utils/Retry.js +21 -24
- package/resolver/app/utils/Retry.js.map +1 -1
- package/resolver/app/utils/sleep.js +4 -5
- package/resolver/app/utils/sleep.js.map +1 -1
- package/resolver/app/validation/body.js +26 -31
- package/resolver/app/validation/body.js.map +1 -1
- package/resolver/app/validation/createJsonTransform.js +12 -13
- package/resolver/app/validation/createJsonTransform.js.map +1 -1
- package/resolver/app/validation/detail.js +30 -35
- package/resolver/app/validation/detail.js.map +1 -1
- package/resolver/app/validation/event.js +17 -18
- package/resolver/app/validation/event.js.map +1 -1
- package/resolver/app/validation/numericString.js +8 -11
- package/resolver/app/validation/numericString.js.map +1 -1
- package/resolver/app/validation/system.js +8 -9
- package/resolver/app/validation/system.js.map +1 -1
- package/resolver/createEventHandlerPlugin.js +70 -99
- package/resolver/createEventHandlerPlugin.js.map +1 -1
- package/resolver/createResolverHandler.js +49 -57
- package/resolver/createResolverHandler.js.map +1 -1
- package/resolver/deployment/Deployment.js +28 -31
- package/resolver/deployment/Deployment.js.map +1 -1
- package/resolver/deployment/Deployments.js +20 -23
- package/resolver/deployment/Deployments.js.map +1 -1
- package/resolver/deployment/DeploymentsFetcher.js +75 -84
- package/resolver/deployment/DeploymentsFetcher.js.map +1 -1
- package/resolver/deployment/types.js +0 -3
- package/resolver/lambda/LambdaTrigger.js +24 -26
- package/resolver/lambda/LambdaTrigger.js.map +1 -1
- package/resolver/plugins/CommandHandlerPlugin.js +16 -15
- package/resolver/plugins/CommandHandlerPlugin.js.map +1 -1
- package/resolver/plugins/StorerAfterEachPlugin.js +20 -19
- package/resolver/plugins/StorerAfterEachPlugin.js.map +1 -1
- package/resolver/plugins/TransformRecordPlugin.js +19 -18
- package/resolver/plugins/TransformRecordPlugin.js.map +1 -1
- package/resolver/recordTypes/fileManager/CopyFile.js +44 -67
- package/resolver/recordTypes/fileManager/CopyFile.js.map +1 -1
- package/resolver/recordTypes/fileManager/DeleteFile.js +40 -58
- package/resolver/recordTypes/fileManager/DeleteFile.js.map +1 -1
- package/resolver/recordTypes/fileManager/fileManager.js +5 -3
- package/resolver/recordTypes/fileManager/fileManager.js.map +1 -1
- package/resolver/recordTypes/fileManager/fileManagerOnDelete.js +24 -35
- package/resolver/recordTypes/fileManager/fileManagerOnDelete.js.map +1 -1
- package/resolver/recordTypes/fileManager/fileManagerOnPut.js +24 -35
- package/resolver/recordTypes/fileManager/fileManagerOnPut.js.map +1 -1
- package/resolver/recordTypes/fileManager/shouldBeHandled.js +10 -27
- package/resolver/recordTypes/fileManager/shouldBeHandled.js.map +1 -1
- package/resolver/recordTypes/fileManager/types.js +0 -3
- package/resolver/recordTypes/users/CopyUser.js +44 -54
- package/resolver/recordTypes/users/CopyUser.js.map +1 -1
- package/resolver/recordTypes/users/DeleteUser.js +40 -51
- package/resolver/recordTypes/users/DeleteUser.js.map +1 -1
- package/resolver/recordTypes/users/shouldBeHandled.js +10 -26
- package/resolver/recordTypes/users/shouldBeHandled.js.map +1 -1
- package/resolver/recordTypes/users/types.js +0 -3
- package/resolver/recordTypes/users/users.js +5 -3
- package/resolver/recordTypes/users/users.js.map +1 -1
- package/resolver/recordTypes/users/usersOnDelete.js +23 -34
- package/resolver/recordTypes/users/usersOnDelete.js.map +1 -1
- package/resolver/recordTypes/users/usersOnPut.js +24 -35
- package/resolver/recordTypes/users/usersOnPut.js.map +1 -1
- package/sync/FilterOutRecord.js +9 -17
- package/sync/FilterOutRecord.js.map +1 -1
- package/sync/attachToDynamoDbDocument.js +46 -65
- package/sync/attachToDynamoDbDocument.js.map +1 -1
- package/sync/createHandler.js +31 -40
- package/sync/createHandler.js.map +1 -1
- package/sync/createSendDataToEventBridgeOnRequestEnd.js +32 -37
- package/sync/createSendDataToEventBridgeOnRequestEnd.js.map +1 -1
- package/sync/createSyncSystem.js +24 -26
- package/sync/createSyncSystem.js.map +1 -1
- package/sync/filter/createDefaultFilterOutRecordPlugins.js +37 -76
- package/sync/filter/createDefaultFilterOutRecordPlugins.js.map +1 -1
- package/sync/handler/Handler.js +64 -76
- package/sync/handler/Handler.js.map +1 -1
- package/sync/handler/HandlerConverter.js +21 -29
- package/sync/handler/HandlerConverter.js.map +1 -1
- package/sync/handler/converter/BatchGetCommandConverter.js +12 -11
- package/sync/handler/converter/BatchGetCommandConverter.js.map +1 -1
- package/sync/handler/converter/BatchWriteCommandConverter.js +12 -11
- package/sync/handler/converter/BatchWriteCommandConverter.js.map +1 -1
- package/sync/handler/converter/DeleteCommandConverter.js +12 -11
- package/sync/handler/converter/DeleteCommandConverter.js.map +1 -1
- package/sync/handler/converter/GetCommandConverter.js +12 -11
- package/sync/handler/converter/GetCommandConverter.js.map +1 -1
- package/sync/handler/converter/PutCommandConverter.js +12 -11
- package/sync/handler/converter/PutCommandConverter.js.map +1 -1
- package/sync/handler/converter/QueryCommandConverter.js +12 -11
- package/sync/handler/converter/QueryCommandConverter.js.map +1 -1
- package/sync/handler/converter/ScanCommandConverter.js +12 -11
- package/sync/handler/converter/ScanCommandConverter.js.map +1 -1
- package/sync/handler/converter/UpdateCommandConverter.js +12 -11
- package/sync/handler/converter/UpdateCommandConverter.js.map +1 -1
- package/sync/handler/converter/commands/BatchWriteCommandValue.js +34 -37
- package/sync/handler/converter/commands/BatchWriteCommandValue.js.map +1 -1
- package/sync/handler/converter/commands/DeleteCommandValue.js +19 -16
- package/sync/handler/converter/commands/DeleteCommandValue.js.map +1 -1
- package/sync/handler/converter/commands/NullCommandValue.js +8 -8
- package/sync/handler/converter/commands/NullCommandValue.js.map +1 -1
- package/sync/handler/converter/commands/PutCommandValue.js +19 -16
- package/sync/handler/converter/commands/PutCommandValue.js.map +1 -1
- package/sync/handler/converter/commands/UpdateCommandValue.js +19 -16
- package/sync/handler/converter/commands/UpdateCommandValue.js.map +1 -1
- package/sync/handler/types.js +0 -3
- package/sync/plugins/FilterOutRecordPlugin.js +17 -19
- package/sync/plugins/FilterOutRecordPlugin.js.map +1 -1
- package/sync/requestPlugin.js +33 -48
- package/sync/requestPlugin.js.map +1 -1
- package/sync/types.js +0 -3
- package/sync/utils/getTableType.js +10 -9
- package/sync/utils/getTableType.js.map +1 -1
- package/sync/utils/manifest.js +28 -35
- package/sync/utils/manifest.js.map +1 -1
- package/sync/utils/validateSystemInput.js +21 -29
- package/sync/utils/validateSystemInput.js.map +1 -1
- package/types.js +6 -5
- package/types.js.map +1 -1
- package/utils/createSystemName.js +5 -3
- package/utils/createSystemName.js.map +1 -1
- package/worker/actions/copyFile/CopyFile.js +183 -258
- package/worker/actions/copyFile/CopyFile.js.map +1 -1
- package/worker/actions/copyFile/copyFileAction.js +26 -34
- package/worker/actions/copyFile/copyFileAction.js.map +1 -1
- package/worker/actions/copyFile/copyFileSchema.js +13 -14
- package/worker/actions/copyFile/copyFileSchema.js.map +1 -1
- package/worker/actions/copyFile/types.js +0 -3
- package/worker/actions/createUser/CreateUser.js +51 -74
- package/worker/actions/createUser/CreateUser.js.map +1 -1
- package/worker/actions/createUser/createUserAction.js +24 -28
- package/worker/actions/createUser/createUserAction.js.map +1 -1
- package/worker/actions/createUser/createUserSchema.js +13 -14
- package/worker/actions/createUser/createUserSchema.js.map +1 -1
- package/worker/actions/createUser/types.js +0 -3
- package/worker/actions/deleteFile/DeleteFile.js +52 -67
- package/worker/actions/deleteFile/DeleteFile.js.map +1 -1
- package/worker/actions/deleteFile/deleteFileAction.js +21 -29
- package/worker/actions/deleteFile/deleteFileAction.js.map +1 -1
- package/worker/actions/deleteFile/deleteFileSchema.js +13 -14
- package/worker/actions/deleteFile/deleteFileSchema.js.map +1 -1
- package/worker/actions/deleteFile/types.js +0 -3
- package/worker/actions/deleteUser/DeleteUser.js +38 -52
- package/worker/actions/deleteUser/DeleteUser.js.map +1 -1
- package/worker/actions/deleteUser/deleteUserAction.js +22 -26
- package/worker/actions/deleteUser/deleteUserAction.js.map +1 -1
- package/worker/actions/deleteUser/deleteUserSchema.js +12 -13
- package/worker/actions/deleteUser/deleteUserSchema.js.map +1 -1
- package/worker/actions/deleteUser/types.js +0 -3
- package/worker/actions/logValidationError.js +5 -6
- package/worker/actions/logValidationError.js.map +1 -1
- package/worker/actions/removeCognitoUserAttributes.js +9 -15
- package/worker/actions/removeCognitoUserAttributes.js.map +1 -1
- package/worker/actions/updateUser/UpdateUser.js +48 -65
- package/worker/actions/updateUser/UpdateUser.js.map +1 -1
- package/worker/actions/updateUser/types.js +0 -3
- package/worker/actions/updateUser/updateUserAction.js +24 -28
- package/worker/actions/updateUser/updateUserAction.js.map +1 -1
- package/worker/actions/updateUser/updateUserSchema.js +13 -14
- package/worker/actions/updateUser/updateUserSchema.js.map +1 -1
- package/worker/createWorkerHandler.js +24 -30
- package/worker/createWorkerHandler.js.map +1 -1
- package/worker/handler/WorkerActionHandler.js +19 -21
- package/worker/handler/WorkerActionHandler.js.map +1 -1
- package/worker/handler/eventHandler.js +14 -16
- package/worker/handler/eventHandler.js.map +1 -1
- package/worker/plugins/WorkerActionPlugin.js +17 -16
- package/worker/plugins/WorkerActionPlugin.js.map +1 -1
- package/worker/types.js +0 -3
- package/index.js.map +0 -1
- package/resolver/app/abstractions/RecordHandler.js.map +0 -1
- package/resolver/app/abstractions/RecordsValidation.js.map +0 -1
- package/resolver/app/abstractions/ResolverApplication.js.map +0 -1
- package/resolver/app/abstractions/ResolverRecord.js.map +0 -1
- package/resolver/app/bundler/types.js.map +0 -1
- package/resolver/app/data/types.js.map +0 -1
- package/resolver/app/fetcher/types.js.map +0 -1
- package/resolver/app/ingestor/types.js.map +0 -1
- package/resolver/app/storer/types.js.map +0 -1
- package/resolver/deployment/types.js.map +0 -1
- package/resolver/recordTypes/fileManager/types.js.map +0 -1
- package/resolver/recordTypes/users/types.js.map +0 -1
- package/sync/handler/types.js.map +0 -1
- package/sync/types.js.map +0 -1
- package/worker/actions/copyFile/types.js.map +0 -1
- package/worker/actions/createUser/types.js.map +0 -1
- package/worker/actions/deleteFile/types.js.map +0 -1
- package/worker/actions/deleteUser/types.js.map +0 -1
- package/worker/actions/updateUser/types.js.map +0 -1
- package/worker/types.js.map +0 -1
|
@@ -1,123 +1,98 @@
|
|
|
1
1
|
import { BatchGetCommand } from "@webiny/aws-sdk/client-dynamodb/index.js";
|
|
2
|
-
import
|
|
2
|
+
import chunk from "lodash/chunk.js";
|
|
3
3
|
import { SourceDataContainer } from "../data/SourceDataContainer.js";
|
|
4
4
|
import { convertException } from "@webiny/utils";
|
|
5
5
|
import { createRetry } from "../utils/Retry.js";
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
}
|
|
12
|
-
async exec(params) {
|
|
13
|
-
const {
|
|
14
|
-
deployment,
|
|
15
|
-
items: input,
|
|
16
|
-
table,
|
|
17
|
-
maxBatchSize = 25
|
|
18
|
-
} = params;
|
|
19
|
-
if (input.length === 0) {
|
|
20
|
-
return {
|
|
21
|
-
items: SourceDataContainer.create()
|
|
22
|
-
};
|
|
6
|
+
class Fetcher {
|
|
7
|
+
constructor(params){
|
|
8
|
+
this.createDocumentClient = params.createDocumentClient;
|
|
9
|
+
this.maxRetries = params.maxRetries || 10;
|
|
10
|
+
this.retryDelay = params.retryDelay || 1000;
|
|
23
11
|
}
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
12
|
+
async exec(params) {
|
|
13
|
+
const { deployment, items: input, table, maxBatchSize = 25 } = params;
|
|
14
|
+
if (0 === input.length) return {
|
|
15
|
+
items: SourceDataContainer.create()
|
|
16
|
+
};
|
|
17
|
+
const client = this.createDocumentClient(deployment);
|
|
18
|
+
const results = await this.execute({
|
|
19
|
+
client,
|
|
20
|
+
table,
|
|
21
|
+
maxBatchSize,
|
|
22
|
+
records: input
|
|
23
|
+
});
|
|
24
|
+
const items = SourceDataContainer.create();
|
|
25
|
+
for (const item of input){
|
|
26
|
+
const data = results.find((result)=>item.PK === result.PK && item.SK === result.SK);
|
|
27
|
+
items.add({
|
|
28
|
+
PK: item.PK,
|
|
29
|
+
SK: item.SK,
|
|
30
|
+
table,
|
|
31
|
+
source: deployment
|
|
32
|
+
}, data || null);
|
|
33
|
+
}
|
|
34
|
+
return {
|
|
35
|
+
items
|
|
36
|
+
};
|
|
42
37
|
}
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
38
|
+
async execute(params) {
|
|
39
|
+
const { client, table, records: items, maxBatchSize } = params;
|
|
40
|
+
const batches = chunk(items, maxBatchSize);
|
|
41
|
+
const results = [];
|
|
42
|
+
for (const batch of batches){
|
|
43
|
+
let keys = this.getKeys(batch);
|
|
44
|
+
while(keys.length > 0){
|
|
45
|
+
const command = new BatchGetCommand({
|
|
46
|
+
RequestItems: {
|
|
47
|
+
[table.name]: {
|
|
48
|
+
Keys: keys
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
});
|
|
52
|
+
const { items, unprocessedKeys } = await this.runCommand({
|
|
53
|
+
command,
|
|
54
|
+
client,
|
|
55
|
+
table: table.name
|
|
56
|
+
});
|
|
57
|
+
results.push(...items);
|
|
58
|
+
keys = unprocessedKeys;
|
|
63
59
|
}
|
|
64
|
-
|
|
60
|
+
}
|
|
61
|
+
return results;
|
|
62
|
+
}
|
|
63
|
+
async runCommand(params) {
|
|
64
|
+
const { command, table, client } = params;
|
|
65
|
+
const retry = createRetry({
|
|
66
|
+
maxRetries: this.maxRetries,
|
|
67
|
+
retryDelay: this.retryDelay
|
|
65
68
|
});
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
69
|
+
return await retry.retry(async ()=>{
|
|
70
|
+
const result = await client.send(command);
|
|
71
|
+
return {
|
|
72
|
+
items: result.Responses?.[table] || [],
|
|
73
|
+
unprocessedKeys: result.UnprocessedKeys?.[table]?.Keys || []
|
|
74
|
+
};
|
|
75
|
+
}, {
|
|
76
|
+
onFail: (ex)=>{
|
|
77
|
+
console.error(`Max retries reached. Could not fetch items from table: ${table}`);
|
|
78
|
+
console.log(convertException(ex));
|
|
79
|
+
console.log(JSON.stringify(command));
|
|
80
|
+
}
|
|
73
81
|
});
|
|
74
|
-
results.push(...items);
|
|
75
|
-
keys = unprocessedKeys;
|
|
76
|
-
}
|
|
77
82
|
}
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
retryDelay: this.retryDelay
|
|
89
|
-
});
|
|
90
|
-
return await retry.retry(async () => {
|
|
91
|
-
const result = await client.send(command);
|
|
92
|
-
return {
|
|
93
|
-
items: result.Responses?.[table] || [],
|
|
94
|
-
unprocessedKeys: result.UnprocessedKeys?.[table]?.Keys || []
|
|
95
|
-
};
|
|
96
|
-
}, {
|
|
97
|
-
onFail: ex => {
|
|
98
|
-
console.error(`Max retries reached. Could not fetch items from table: ${table}`);
|
|
99
|
-
console.log(convertException(ex));
|
|
100
|
-
console.log(JSON.stringify(command));
|
|
101
|
-
}
|
|
102
|
-
});
|
|
103
|
-
}
|
|
104
|
-
getKeys(items) {
|
|
105
|
-
const keys = new Map();
|
|
106
|
-
for (const item of items) {
|
|
107
|
-
const key = `PK:${item.PK}#SK:${item.SK}`;
|
|
108
|
-
if (keys.has(key)) {
|
|
109
|
-
continue;
|
|
110
|
-
}
|
|
111
|
-
keys.set(key, {
|
|
112
|
-
PK: item.PK,
|
|
113
|
-
SK: item.SK
|
|
114
|
-
});
|
|
83
|
+
getKeys(items) {
|
|
84
|
+
const keys = new Map();
|
|
85
|
+
for (const item of items){
|
|
86
|
+
const key = `PK:${item.PK}#SK:${item.SK}`;
|
|
87
|
+
if (!keys.has(key)) keys.set(key, {
|
|
88
|
+
PK: item.PK,
|
|
89
|
+
SK: item.SK
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
return Array.from(keys.values());
|
|
115
93
|
}
|
|
116
|
-
return Array.from(keys.values());
|
|
117
|
-
}
|
|
118
94
|
}
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
};
|
|
95
|
+
const createFetcher = (params)=>new Fetcher(params);
|
|
96
|
+
export { Fetcher, createFetcher };
|
|
122
97
|
|
|
123
98
|
//# sourceMappingURL=Fetcher.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"
|
|
1
|
+
{"version":3,"file":"resolver/app/fetcher/Fetcher.js","sources":["../../../../src/resolver/app/fetcher/Fetcher.ts"],"sourcesContent":["import type { IFetcher, IFetcherExecParams, IFetcherExecResult } from \"./types.js\";\nimport type { DynamoDBDocument } from \"@webiny/aws-sdk/client-dynamodb/index.js\";\nimport { BatchGetCommand } from \"@webiny/aws-sdk/client-dynamodb/index.js\";\nimport lodashChunk from \"lodash/chunk.js\";\nimport type { IDeployment } from \"~/resolver/deployment/types.js\";\nimport { SourceDataContainer } from \"~/resolver/app/data/SourceDataContainer.js\";\nimport type { GenericRecord } from \"@webiny/api/types.js\";\nimport { convertException } from \"@webiny/utils\";\nimport type { IStoreItem } from \"~/resolver/app/storer/types.js\";\nimport type { ITable } from \"~/sync/types.js\";\nimport { createRetry } from \"~/resolver/app/utils/Retry.js\";\n\nexport interface IFetcherParamsCreateDocumentClientCallable {\n (deployment: Pick<IDeployment, \"region\">): Pick<DynamoDBDocument, \"send\">;\n}\n\nexport interface IFetcherParams {\n maxRetries?: number;\n retryDelay?: number;\n createDocumentClient: IFetcherParamsCreateDocumentClientCallable;\n}\n\ninterface IKeys {\n PK: string;\n SK: string;\n}\n\ninterface IFetcherExecuteRunCommandParams {\n command: BatchGetCommand;\n table: string;\n client: Pick<DynamoDBDocument, \"send\">;\n}\n\ninterface IFetcherExecuteRunCommandResult<T = GenericRecord> {\n items: T[];\n unprocessedKeys: IKeys[];\n}\n\ninterface IFetchExecuteExecuteParamsItem {\n PK: string;\n SK: string;\n}\n\ninterface IFetchExecuteExecuteParams {\n maxBatchSize: number;\n client: Pick<DynamoDBDocument, \"send\">;\n table: ITable;\n records: IFetchExecuteExecuteParamsItem[];\n}\n\nexport class Fetcher implements IFetcher {\n private readonly createDocumentClient: IFetcherParamsCreateDocumentClientCallable;\n private readonly maxRetries: number;\n private readonly retryDelay: number;\n\n public constructor(params: IFetcherParams) {\n this.createDocumentClient = params.createDocumentClient;\n this.maxRetries = params.maxRetries || 10;\n this.retryDelay = params.retryDelay || 1000;\n }\n\n public async exec(params: IFetcherExecParams): Promise<IFetcherExecResult> {\n const { deployment, items: input, table, maxBatchSize = 25 } = params;\n if (input.length === 0) {\n return {\n items: SourceDataContainer.create()\n };\n }\n\n const client = this.createDocumentClient(deployment);\n\n const results = await this.execute({\n client,\n table,\n maxBatchSize,\n records: input\n });\n\n const items = SourceDataContainer.create();\n\n for (const item of input) {\n const data = results.find(result => {\n return item.PK === result.PK && item.SK === result.SK;\n });\n items.add(\n {\n PK: item.PK,\n SK: item.SK,\n table,\n source: deployment\n },\n data || null\n );\n }\n\n return {\n items\n };\n }\n\n private async execute<T = IStoreItem>(params: IFetchExecuteExecuteParams) {\n const { client, table, records: items, maxBatchSize } = params;\n const batches = lodashChunk(items, maxBatchSize);\n\n const results: T[] = [];\n for (const batch of batches) {\n let keys = this.getKeys(batch);\n\n while (keys.length > 0) {\n const command = new BatchGetCommand({\n RequestItems: {\n [table.name]: {\n Keys: keys\n }\n }\n });\n const { items, unprocessedKeys } = await this.runCommand<T>({\n command,\n client,\n table: table.name\n });\n\n results.push(...items);\n\n keys = unprocessedKeys;\n }\n }\n return results;\n }\n\n private async runCommand<T = GenericRecord>(\n params: IFetcherExecuteRunCommandParams\n ): Promise<IFetcherExecuteRunCommandResult<T>> {\n const { command, table, client } = params;\n\n const retry = createRetry({\n maxRetries: this.maxRetries,\n retryDelay: this.retryDelay\n });\n\n return await retry.retry(\n async () => {\n const result = await client.send(command);\n\n return {\n items: (result.Responses?.[table] || []) as T[],\n unprocessedKeys: (result.UnprocessedKeys?.[table]?.Keys || []) as IKeys[]\n };\n },\n {\n onFail: ex => {\n console.error(\n `Max retries reached. Could not fetch items from table: ${table}`\n );\n console.log(convertException(ex));\n console.log(JSON.stringify(command));\n }\n }\n );\n }\n\n private getKeys(items: IFetchExecuteExecuteParamsItem[]): IKeys[] {\n const keys = new Map<string, IKeys>();\n\n for (const item of items) {\n const key = `PK:${item.PK}#SK:${item.SK}`;\n if (keys.has(key)) {\n continue;\n }\n keys.set(key, {\n PK: item.PK,\n SK: item.SK\n });\n }\n\n return Array.from(keys.values());\n }\n}\n\nexport const createFetcher = (params: IFetcherParams): IFetcher => {\n return new Fetcher(params);\n};\n"],"names":["Fetcher","params","deployment","input","table","maxBatchSize","SourceDataContainer","client","results","items","item","data","result","batches","lodashChunk","batch","keys","command","BatchGetCommand","unprocessedKeys","retry","createRetry","ex","console","convertException","JSON","Map","key","Array","createFetcher"],"mappings":";;;;;AAkDO,MAAMA;IAKT,YAAmBC,MAAsB,CAAE;QACvC,IAAI,CAAC,oBAAoB,GAAGA,OAAO,oBAAoB;QACvD,IAAI,CAAC,UAAU,GAAGA,OAAO,UAAU,IAAI;QACvC,IAAI,CAAC,UAAU,GAAGA,OAAO,UAAU,IAAI;IAC3C;IAEA,MAAa,KAAKA,MAA0B,EAA+B;QACvE,MAAM,EAAEC,UAAU,EAAE,OAAOC,KAAK,EAAEC,KAAK,EAAEC,eAAe,EAAE,EAAE,GAAGJ;QAC/D,IAAIE,AAAiB,MAAjBA,MAAM,MAAM,EACZ,OAAO;YACH,OAAOG,oBAAoB,MAAM;QACrC;QAGJ,MAAMC,SAAS,IAAI,CAAC,oBAAoB,CAACL;QAEzC,MAAMM,UAAU,MAAM,IAAI,CAAC,OAAO,CAAC;YAC/BD;YACAH;YACAC;YACA,SAASF;QACb;QAEA,MAAMM,QAAQH,oBAAoB,MAAM;QAExC,KAAK,MAAMI,QAAQP,MAAO;YACtB,MAAMQ,OAAOH,QAAQ,IAAI,CAACI,CAAAA,SACfF,KAAK,EAAE,KAAKE,OAAO,EAAE,IAAIF,KAAK,EAAE,KAAKE,OAAO,EAAE;YAEzDH,MAAM,GAAG,CACL;gBACI,IAAIC,KAAK,EAAE;gBACX,IAAIA,KAAK,EAAE;gBACXN;gBACA,QAAQF;YACZ,GACAS,QAAQ;QAEhB;QAEA,OAAO;YACHF;QACJ;IACJ;IAEA,MAAc,QAAwBR,MAAkC,EAAE;QACtE,MAAM,EAAEM,MAAM,EAAEH,KAAK,EAAE,SAASK,KAAK,EAAEJ,YAAY,EAAE,GAAGJ;QACxD,MAAMY,UAAUC,MAAYL,OAAOJ;QAEnC,MAAMG,UAAe,EAAE;QACvB,KAAK,MAAMO,SAASF,QAAS;YACzB,IAAIG,OAAO,IAAI,CAAC,OAAO,CAACD;YAExB,MAAOC,KAAK,MAAM,GAAG,EAAG;gBACpB,MAAMC,UAAU,IAAIC,gBAAgB;oBAChC,cAAc;wBACV,CAACd,MAAM,IAAI,CAAC,EAAE;4BACV,MAAMY;wBACV;oBACJ;gBACJ;gBACA,MAAM,EAAEP,KAAK,EAAEU,eAAe,EAAE,GAAG,MAAM,IAAI,CAAC,UAAU,CAAI;oBACxDF;oBACAV;oBACA,OAAOH,MAAM,IAAI;gBACrB;gBAEAI,QAAQ,IAAI,IAAIC;gBAEhBO,OAAOG;YACX;QACJ;QACA,OAAOX;IACX;IAEA,MAAc,WACVP,MAAuC,EACI;QAC3C,MAAM,EAAEgB,OAAO,EAAEb,KAAK,EAAEG,MAAM,EAAE,GAAGN;QAEnC,MAAMmB,QAAQC,YAAY;YACtB,YAAY,IAAI,CAAC,UAAU;YAC3B,YAAY,IAAI,CAAC,UAAU;QAC/B;QAEA,OAAO,MAAMD,MAAM,KAAK,CACpB;YACI,MAAMR,SAAS,MAAML,OAAO,IAAI,CAACU;YAEjC,OAAO;gBACH,OAAQL,OAAO,SAAS,EAAE,CAACR,MAAM,IAAI,EAAE;gBACvC,iBAAkBQ,OAAO,eAAe,EAAE,CAACR,MAAM,EAAE,QAAQ,EAAE;YACjE;QACJ,GACA;YACI,QAAQkB,CAAAA;gBACJC,QAAQ,KAAK,CACT,CAAC,uDAAuD,EAAEnB,OAAO;gBAErEmB,QAAQ,GAAG,CAACC,iBAAiBF;gBAC7BC,QAAQ,GAAG,CAACE,KAAK,SAAS,CAACR;YAC/B;QACJ;IAER;IAEQ,QAAQR,KAAuC,EAAW;QAC9D,MAAMO,OAAO,IAAIU;QAEjB,KAAK,MAAMhB,QAAQD,MAAO;YACtB,MAAMkB,MAAM,CAAC,GAAG,EAAEjB,KAAK,EAAE,CAAC,IAAI,EAAEA,KAAK,EAAE,EAAE;YACzC,KAAIM,KAAK,GAAG,CAACW,MAGbX,KAAK,GAAG,CAACW,KAAK;gBACV,IAAIjB,KAAK,EAAE;gBACX,IAAIA,KAAK,EAAE;YACf;QACJ;QAEA,OAAOkB,MAAM,IAAI,CAACZ,KAAK,MAAM;IACjC;AACJ;AAEO,MAAMa,gBAAgB,CAAC5B,SACnB,IAAID,QAAQC"}
|
|
@@ -1,36 +1,31 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
});
|
|
27
|
-
}
|
|
1
|
+
class Ingestor {
|
|
2
|
+
constructor(params){
|
|
3
|
+
this.createIngestorResult = params.createIngestorResult;
|
|
4
|
+
this.getSource = params.getSource;
|
|
5
|
+
}
|
|
6
|
+
async ingest(params) {
|
|
7
|
+
const { records } = params;
|
|
8
|
+
const result = this.createIngestorResult();
|
|
9
|
+
for (const record of records){
|
|
10
|
+
const detail = record.body.detail;
|
|
11
|
+
const source = this.getSource(detail.source.name);
|
|
12
|
+
if (!source) {
|
|
13
|
+
console.error(`Could not find deployment for SQS Record source: ${detail.source.name}. More info in next log line.`);
|
|
14
|
+
console.log(JSON.stringify({
|
|
15
|
+
source: detail.source,
|
|
16
|
+
items: detail.items
|
|
17
|
+
}));
|
|
18
|
+
continue;
|
|
19
|
+
}
|
|
20
|
+
for (const item of detail.items)result.add({
|
|
21
|
+
item,
|
|
22
|
+
source
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
return result;
|
|
28
26
|
}
|
|
29
|
-
return result;
|
|
30
|
-
}
|
|
31
27
|
}
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
};
|
|
28
|
+
const createIngestor = (params)=>new Ingestor(params);
|
|
29
|
+
export { Ingestor, createIngestor };
|
|
35
30
|
|
|
36
31
|
//# sourceMappingURL=Ingestor.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"
|
|
1
|
+
{"version":3,"file":"resolver/app/ingestor/Ingestor.js","sources":["../../../../src/resolver/app/ingestor/Ingestor.ts"],"sourcesContent":["import type { IIngestor, IIngestorIngestParams, IIngestorResult } from \"./types.js\";\nimport type { IDeployment } from \"~/resolver/deployment/types.js\";\n\nexport interface IIngestorParams {\n createIngestorResult: () => IIngestorResult;\n getSource: (name: string) => IDeployment | null;\n}\n\nexport class Ingestor implements IIngestor {\n private readonly createIngestorResult: () => IIngestorResult;\n private readonly getSource: (name: string) => IDeployment | null;\n\n public constructor(params: IIngestorParams) {\n this.createIngestorResult = params.createIngestorResult;\n this.getSource = params.getSource;\n }\n\n public async ingest(params: IIngestorIngestParams): Promise<IIngestorResult> {\n const { records } = params;\n\n const result = this.createIngestorResult();\n\n for (const record of records) {\n const detail = record.body.detail;\n const source = this.getSource(detail.source.name);\n if (!source) {\n console.error(\n `Could not find deployment for SQS Record source: ${detail.source.name}. More info in next log line.`\n );\n console.log(\n JSON.stringify({\n source: detail.source,\n items: detail.items\n })\n );\n continue;\n }\n\n for (const item of detail.items) {\n result.add({\n item,\n source\n });\n }\n }\n\n return result;\n }\n}\n\nexport const createIngestor = (params: IIngestorParams): IIngestor => {\n return new Ingestor(params);\n};\n"],"names":["Ingestor","params","records","result","record","detail","source","console","JSON","item","createIngestor"],"mappings":"AAQO,MAAMA;IAIT,YAAmBC,MAAuB,CAAE;QACxC,IAAI,CAAC,oBAAoB,GAAGA,OAAO,oBAAoB;QACvD,IAAI,CAAC,SAAS,GAAGA,OAAO,SAAS;IACrC;IAEA,MAAa,OAAOA,MAA6B,EAA4B;QACzE,MAAM,EAAEC,OAAO,EAAE,GAAGD;QAEpB,MAAME,SAAS,IAAI,CAAC,oBAAoB;QAExC,KAAK,MAAMC,UAAUF,QAAS;YAC1B,MAAMG,SAASD,OAAO,IAAI,CAAC,MAAM;YACjC,MAAME,SAAS,IAAI,CAAC,SAAS,CAACD,OAAO,MAAM,CAAC,IAAI;YAChD,IAAI,CAACC,QAAQ;gBACTC,QAAQ,KAAK,CACT,CAAC,iDAAiD,EAAEF,OAAO,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC;gBAEzGE,QAAQ,GAAG,CACPC,KAAK,SAAS,CAAC;oBACX,QAAQH,OAAO,MAAM;oBACrB,OAAOA,OAAO,KAAK;gBACvB;gBAEJ;YACJ;YAEA,KAAK,MAAMI,QAAQJ,OAAO,KAAK,CAC3BF,OAAO,GAAG,CAAC;gBACPM;gBACAH;YACJ;QAER;QAEA,OAAOH;IACX;AACJ;AAEO,MAAMO,iBAAiB,CAACT,SACpB,IAAID,SAASC"}
|
|
@@ -1,37 +1,33 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
1
|
+
class IngestorResult {
|
|
2
|
+
add(params) {
|
|
3
|
+
const { item, source } = params;
|
|
4
|
+
let table;
|
|
5
|
+
try {
|
|
6
|
+
table = source.getTable(item.tableType);
|
|
7
|
+
} catch {
|
|
8
|
+
console.error(`Could not find table for SQS Record source: ${item.tableName} / ${item.tableType}. More info in next log line.`);
|
|
9
|
+
console.log(JSON.stringify(item));
|
|
10
|
+
return;
|
|
11
|
+
}
|
|
12
|
+
const command = "delete" === item.command ? "delete" : "put";
|
|
13
|
+
const key = `PK:${item.PK}#SK:${item.SK}#cmd:${command}#source:${source.name}`;
|
|
14
|
+
if (this.items.has(key)) return;
|
|
15
|
+
this.items.set(key, {
|
|
16
|
+
PK: item.PK,
|
|
17
|
+
SK: item.SK,
|
|
18
|
+
source,
|
|
19
|
+
command,
|
|
20
|
+
table
|
|
21
|
+
});
|
|
15
22
|
}
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
23
|
+
getItems() {
|
|
24
|
+
return Array.from(this.items.values());
|
|
25
|
+
}
|
|
26
|
+
constructor(){
|
|
27
|
+
this.items = new Map();
|
|
20
28
|
}
|
|
21
|
-
this.items.set(key, {
|
|
22
|
-
PK: item.PK,
|
|
23
|
-
SK: item.SK,
|
|
24
|
-
source,
|
|
25
|
-
command,
|
|
26
|
-
table
|
|
27
|
-
});
|
|
28
|
-
}
|
|
29
|
-
getItems() {
|
|
30
|
-
return Array.from(this.items.values());
|
|
31
|
-
}
|
|
32
29
|
}
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
};
|
|
30
|
+
const createIngestorResult = ()=>new IngestorResult();
|
|
31
|
+
export { IngestorResult, createIngestorResult };
|
|
36
32
|
|
|
37
33
|
//# sourceMappingURL=IngestorResult.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"
|
|
1
|
+
{"version":3,"file":"resolver/app/ingestor/IngestorResult.js","sources":["../../../../src/resolver/app/ingestor/IngestorResult.ts"],"sourcesContent":["import type { IIngestorResult, IIngestorResultAddParams, IIngestorResultItem } from \"./types.js\";\nimport type { ITable } from \"~/sync/types.js\";\n\nexport class IngestorResult implements IIngestorResult {\n private readonly items: Map<string, IIngestorResultItem> = new Map();\n\n public add(params: IIngestorResultAddParams): void {\n const { item, source } = params;\n let table: ITable;\n try {\n table = source.getTable(item.tableType);\n } catch {\n console.error(\n `Could not find table for SQS Record source: ${item.tableName} / ${item.tableType}. More info in next log line.`\n );\n console.log(JSON.stringify(item));\n return;\n }\n\n const command = item.command === \"delete\" ? \"delete\" : \"put\";\n\n const key = `PK:${item.PK}#SK:${item.SK}#cmd:${command}#source:${source.name}`;\n\n if (this.items.has(key)) {\n return;\n }\n\n this.items.set(key, {\n PK: item.PK,\n SK: item.SK,\n source,\n command,\n table\n });\n }\n\n public getItems(): IIngestorResultItem[] {\n return Array.from(this.items.values());\n }\n}\n\nexport const createIngestorResult = (): IIngestorResult => {\n return new IngestorResult();\n};\n"],"names":["IngestorResult","params","item","source","table","console","JSON","command","key","Array","Map","createIngestorResult"],"mappings":"AAGO,MAAMA;IAGF,IAAIC,MAAgC,EAAQ;QAC/C,MAAM,EAAEC,IAAI,EAAEC,MAAM,EAAE,GAAGF;QACzB,IAAIG;QACJ,IAAI;YACAA,QAAQD,OAAO,QAAQ,CAACD,KAAK,SAAS;QAC1C,EAAE,OAAM;YACJG,QAAQ,KAAK,CACT,CAAC,4CAA4C,EAAEH,KAAK,SAAS,CAAC,GAAG,EAAEA,KAAK,SAAS,CAAC,6BAA6B,CAAC;YAEpHG,QAAQ,GAAG,CAACC,KAAK,SAAS,CAACJ;YAC3B;QACJ;QAEA,MAAMK,UAAUL,AAAiB,aAAjBA,KAAK,OAAO,GAAgB,WAAW;QAEvD,MAAMM,MAAM,CAAC,GAAG,EAAEN,KAAK,EAAE,CAAC,IAAI,EAAEA,KAAK,EAAE,CAAC,KAAK,EAAEK,QAAQ,QAAQ,EAAEJ,OAAO,IAAI,EAAE;QAE9E,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAACK,MACf;QAGJ,IAAI,CAAC,KAAK,CAAC,GAAG,CAACA,KAAK;YAChB,IAAIN,KAAK,EAAE;YACX,IAAIA,KAAK,EAAE;YACXC;YACAI;YACAH;QACJ;IACJ;IAEO,WAAkC;QACrC,OAAOK,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM;IACvC;;aAlCiB,KAAK,GAAqC,IAAIC;;AAmCnE;AAEO,MAAMC,uBAAuB,IACzB,IAAIX"}
|
|
@@ -1,95 +1,81 @@
|
|
|
1
1
|
import { DeleteCommand, PutCommand } from "@webiny/aws-sdk/client-dynamodb/index.js";
|
|
2
2
|
import { createRetry } from "../utils/Retry.js";
|
|
3
3
|
import { convertException } from "@webiny/utils";
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
}
|
|
12
|
-
async store(params) {
|
|
13
|
-
const {
|
|
14
|
-
deployment,
|
|
15
|
-
bundle,
|
|
16
|
-
table,
|
|
17
|
-
command,
|
|
18
|
-
items
|
|
19
|
-
} = params;
|
|
20
|
-
if (items.length === 0) {
|
|
21
|
-
return;
|
|
4
|
+
class Storer {
|
|
5
|
+
constructor(params){
|
|
6
|
+
this.maxRetries = params.maxRetries || 10;
|
|
7
|
+
this.retryDelay = params.retryDelay || 1000;
|
|
8
|
+
this.createDocumentClient = params.createDocumentClient;
|
|
9
|
+
this.afterEach = params.afterEach;
|
|
10
|
+
this.onError = params.onError;
|
|
22
11
|
}
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
12
|
+
async store(params) {
|
|
13
|
+
const { deployment, bundle, table, command, items } = params;
|
|
14
|
+
if (0 === items.length) return;
|
|
15
|
+
const client = this.createDocumentClient({
|
|
16
|
+
region: deployment.region
|
|
17
|
+
});
|
|
18
|
+
const retry = createRetry({
|
|
19
|
+
maxRetries: this.maxRetries,
|
|
20
|
+
retryDelay: this.retryDelay
|
|
21
|
+
});
|
|
22
|
+
for (const item of items){
|
|
23
|
+
const result = await retry.retry(async ()=>{
|
|
24
|
+
switch(command){
|
|
25
|
+
case "delete":
|
|
26
|
+
return await client.send(new DeleteCommand({
|
|
27
|
+
TableName: table.name,
|
|
28
|
+
Key: {
|
|
29
|
+
PK: item.PK,
|
|
30
|
+
SK: item.SK
|
|
31
|
+
}
|
|
32
|
+
}));
|
|
33
|
+
case "put":
|
|
34
|
+
return await client.send(new PutCommand({
|
|
35
|
+
TableName: table.name,
|
|
36
|
+
Item: item
|
|
37
|
+
}));
|
|
38
|
+
default:
|
|
39
|
+
console.error(`Unsupported command type: ${command}`);
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
}, {
|
|
43
|
+
onFail: async (error)=>{
|
|
44
|
+
console.error("Error executing batch write command.");
|
|
45
|
+
console.log(convertException(error));
|
|
46
|
+
if (!this.onError) return;
|
|
47
|
+
try {
|
|
48
|
+
await this.onError({
|
|
49
|
+
item,
|
|
50
|
+
command,
|
|
51
|
+
table,
|
|
52
|
+
error
|
|
53
|
+
});
|
|
54
|
+
} catch (ex) {
|
|
55
|
+
console.error(`Error in onError callback for command: ${command}`);
|
|
56
|
+
console.log({
|
|
57
|
+
original: convertException(error),
|
|
58
|
+
error: convertException(ex)
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
}
|
|
69
62
|
});
|
|
70
|
-
|
|
63
|
+
if (result && this.afterEach) try {
|
|
64
|
+
await this.afterEach({
|
|
65
|
+
table,
|
|
66
|
+
command,
|
|
67
|
+
item,
|
|
68
|
+
source: bundle.source,
|
|
69
|
+
target: deployment
|
|
70
|
+
});
|
|
71
|
+
} catch (ex) {
|
|
72
|
+
console.error(`Error in afterEach callback for command: ${command}`);
|
|
73
|
+
console.log(convertException(ex));
|
|
74
|
+
}
|
|
71
75
|
}
|
|
72
|
-
});
|
|
73
|
-
if (!result || !this.afterEach) {
|
|
74
|
-
continue;
|
|
75
|
-
}
|
|
76
|
-
try {
|
|
77
|
-
await this.afterEach({
|
|
78
|
-
table,
|
|
79
|
-
command,
|
|
80
|
-
item,
|
|
81
|
-
source: bundle.source,
|
|
82
|
-
target: deployment
|
|
83
|
-
});
|
|
84
|
-
} catch (ex) {
|
|
85
|
-
console.error(`Error in afterEach callback for command: ${command}`);
|
|
86
|
-
console.log(convertException(ex));
|
|
87
|
-
}
|
|
88
76
|
}
|
|
89
|
-
}
|
|
90
77
|
}
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
};
|
|
78
|
+
const createStorer = (params)=>new Storer(params);
|
|
79
|
+
export { Storer, createStorer };
|
|
94
80
|
|
|
95
81
|
//# sourceMappingURL=Storer.js.map
|