@webiny/api-sync-system 6.3.0 → 6.4.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (239) hide show
  1. package/constants.js +2 -1
  2. package/constants.js.map +1 -1
  3. package/index.js +0 -2
  4. package/package.json +14 -14
  5. package/resolver/app/RecordHandler.js +83 -105
  6. package/resolver/app/RecordHandler.js.map +1 -1
  7. package/resolver/app/RecordsValidation.js +11 -14
  8. package/resolver/app/RecordsValidation.js.map +1 -1
  9. package/resolver/app/ResolverApplication.js +24 -31
  10. package/resolver/app/ResolverApplication.js.map +1 -1
  11. package/resolver/app/abstractions/RecordHandler.js +0 -3
  12. package/resolver/app/abstractions/RecordsValidation.js +0 -3
  13. package/resolver/app/abstractions/ResolverApplication.js +0 -3
  14. package/resolver/app/abstractions/ResolverRecord.js +0 -3
  15. package/resolver/app/bundler/BaseBundle.js +17 -16
  16. package/resolver/app/bundler/BaseBundle.js.map +1 -1
  17. package/resolver/app/bundler/Bundler.js +13 -18
  18. package/resolver/app/bundler/Bundler.js.map +1 -1
  19. package/resolver/app/bundler/Bundles.js +21 -26
  20. package/resolver/app/bundler/Bundles.js.map +1 -1
  21. package/resolver/app/bundler/CommandBundle.js +7 -16
  22. package/resolver/app/bundler/CommandBundle.js.map +1 -1
  23. package/resolver/app/bundler/TableBundle.js +6 -13
  24. package/resolver/app/bundler/TableBundle.js.map +1 -1
  25. package/resolver/app/bundler/types.js +0 -3
  26. package/resolver/app/commandHandler/DeleteCommandHandler.js +22 -29
  27. package/resolver/app/commandHandler/DeleteCommandHandler.js.map +1 -1
  28. package/resolver/app/commandHandler/PutCommandHandler.js +16 -22
  29. package/resolver/app/commandHandler/PutCommandHandler.js.map +1 -1
  30. package/resolver/app/commandHandler/delete.js +11 -12
  31. package/resolver/app/commandHandler/delete.js.map +1 -1
  32. package/resolver/app/commandHandler/put.js +11 -12
  33. package/resolver/app/commandHandler/put.js.map +1 -1
  34. package/resolver/app/data/SourceDataContainer.js +28 -30
  35. package/resolver/app/data/SourceDataContainer.js.map +1 -1
  36. package/resolver/app/data/types.js +0 -3
  37. package/resolver/app/fetcher/Fetcher.js +84 -109
  38. package/resolver/app/fetcher/Fetcher.js.map +1 -1
  39. package/resolver/app/fetcher/types.js +0 -3
  40. package/resolver/app/ingestor/Ingestor.js +27 -32
  41. package/resolver/app/ingestor/Ingestor.js.map +1 -1
  42. package/resolver/app/ingestor/IngestorResult.js +28 -32
  43. package/resolver/app/ingestor/IngestorResult.js.map +1 -1
  44. package/resolver/app/ingestor/types.js +0 -3
  45. package/resolver/app/storer/Storer.js +71 -85
  46. package/resolver/app/storer/Storer.js.map +1 -1
  47. package/resolver/app/storer/types.js +0 -3
  48. package/resolver/app/transform/TransformHandler.js +45 -56
  49. package/resolver/app/transform/TransformHandler.js.map +1 -1
  50. package/resolver/app/transform/middleware.js +13 -20
  51. package/resolver/app/transform/middleware.js.map +1 -1
  52. package/resolver/app/utils/Retry.js +21 -24
  53. package/resolver/app/utils/Retry.js.map +1 -1
  54. package/resolver/app/utils/sleep.js +4 -5
  55. package/resolver/app/utils/sleep.js.map +1 -1
  56. package/resolver/app/validation/body.js +26 -31
  57. package/resolver/app/validation/body.js.map +1 -1
  58. package/resolver/app/validation/createJsonTransform.js +12 -13
  59. package/resolver/app/validation/createJsonTransform.js.map +1 -1
  60. package/resolver/app/validation/detail.js +30 -35
  61. package/resolver/app/validation/detail.js.map +1 -1
  62. package/resolver/app/validation/event.js +17 -18
  63. package/resolver/app/validation/event.js.map +1 -1
  64. package/resolver/app/validation/numericString.js +8 -11
  65. package/resolver/app/validation/numericString.js.map +1 -1
  66. package/resolver/app/validation/system.js +8 -9
  67. package/resolver/app/validation/system.js.map +1 -1
  68. package/resolver/createEventHandlerPlugin.js +70 -99
  69. package/resolver/createEventHandlerPlugin.js.map +1 -1
  70. package/resolver/createResolverHandler.js +49 -57
  71. package/resolver/createResolverHandler.js.map +1 -1
  72. package/resolver/deployment/Deployment.js +28 -31
  73. package/resolver/deployment/Deployment.js.map +1 -1
  74. package/resolver/deployment/Deployments.js +20 -23
  75. package/resolver/deployment/Deployments.js.map +1 -1
  76. package/resolver/deployment/DeploymentsFetcher.js +75 -84
  77. package/resolver/deployment/DeploymentsFetcher.js.map +1 -1
  78. package/resolver/deployment/types.js +0 -3
  79. package/resolver/lambda/LambdaTrigger.js +24 -26
  80. package/resolver/lambda/LambdaTrigger.js.map +1 -1
  81. package/resolver/plugins/CommandHandlerPlugin.js +16 -15
  82. package/resolver/plugins/CommandHandlerPlugin.js.map +1 -1
  83. package/resolver/plugins/StorerAfterEachPlugin.js +20 -19
  84. package/resolver/plugins/StorerAfterEachPlugin.js.map +1 -1
  85. package/resolver/plugins/TransformRecordPlugin.js +19 -18
  86. package/resolver/plugins/TransformRecordPlugin.js.map +1 -1
  87. package/resolver/recordTypes/fileManager/CopyFile.js +44 -67
  88. package/resolver/recordTypes/fileManager/CopyFile.js.map +1 -1
  89. package/resolver/recordTypes/fileManager/DeleteFile.js +40 -58
  90. package/resolver/recordTypes/fileManager/DeleteFile.js.map +1 -1
  91. package/resolver/recordTypes/fileManager/fileManager.js +5 -3
  92. package/resolver/recordTypes/fileManager/fileManager.js.map +1 -1
  93. package/resolver/recordTypes/fileManager/fileManagerOnDelete.js +24 -35
  94. package/resolver/recordTypes/fileManager/fileManagerOnDelete.js.map +1 -1
  95. package/resolver/recordTypes/fileManager/fileManagerOnPut.js +24 -35
  96. package/resolver/recordTypes/fileManager/fileManagerOnPut.js.map +1 -1
  97. package/resolver/recordTypes/fileManager/shouldBeHandled.js +10 -27
  98. package/resolver/recordTypes/fileManager/shouldBeHandled.js.map +1 -1
  99. package/resolver/recordTypes/fileManager/types.js +0 -3
  100. package/resolver/recordTypes/users/CopyUser.js +44 -54
  101. package/resolver/recordTypes/users/CopyUser.js.map +1 -1
  102. package/resolver/recordTypes/users/DeleteUser.js +40 -51
  103. package/resolver/recordTypes/users/DeleteUser.js.map +1 -1
  104. package/resolver/recordTypes/users/shouldBeHandled.js +10 -26
  105. package/resolver/recordTypes/users/shouldBeHandled.js.map +1 -1
  106. package/resolver/recordTypes/users/types.js +0 -3
  107. package/resolver/recordTypes/users/users.js +5 -3
  108. package/resolver/recordTypes/users/users.js.map +1 -1
  109. package/resolver/recordTypes/users/usersOnDelete.js +23 -34
  110. package/resolver/recordTypes/users/usersOnDelete.js.map +1 -1
  111. package/resolver/recordTypes/users/usersOnPut.js +24 -35
  112. package/resolver/recordTypes/users/usersOnPut.js.map +1 -1
  113. package/sync/FilterOutRecord.js +9 -17
  114. package/sync/FilterOutRecord.js.map +1 -1
  115. package/sync/attachToDynamoDbDocument.js +46 -65
  116. package/sync/attachToDynamoDbDocument.js.map +1 -1
  117. package/sync/createHandler.js +31 -40
  118. package/sync/createHandler.js.map +1 -1
  119. package/sync/createSendDataToEventBridgeOnRequestEnd.js +32 -37
  120. package/sync/createSendDataToEventBridgeOnRequestEnd.js.map +1 -1
  121. package/sync/createSyncSystem.js +24 -26
  122. package/sync/createSyncSystem.js.map +1 -1
  123. package/sync/filter/createDefaultFilterOutRecordPlugins.js +37 -76
  124. package/sync/filter/createDefaultFilterOutRecordPlugins.js.map +1 -1
  125. package/sync/handler/Handler.js +64 -76
  126. package/sync/handler/Handler.js.map +1 -1
  127. package/sync/handler/HandlerConverter.js +21 -29
  128. package/sync/handler/HandlerConverter.js.map +1 -1
  129. package/sync/handler/converter/BatchGetCommandConverter.js +12 -11
  130. package/sync/handler/converter/BatchGetCommandConverter.js.map +1 -1
  131. package/sync/handler/converter/BatchWriteCommandConverter.js +12 -11
  132. package/sync/handler/converter/BatchWriteCommandConverter.js.map +1 -1
  133. package/sync/handler/converter/DeleteCommandConverter.js +12 -11
  134. package/sync/handler/converter/DeleteCommandConverter.js.map +1 -1
  135. package/sync/handler/converter/GetCommandConverter.js +12 -11
  136. package/sync/handler/converter/GetCommandConverter.js.map +1 -1
  137. package/sync/handler/converter/PutCommandConverter.js +12 -11
  138. package/sync/handler/converter/PutCommandConverter.js.map +1 -1
  139. package/sync/handler/converter/QueryCommandConverter.js +12 -11
  140. package/sync/handler/converter/QueryCommandConverter.js.map +1 -1
  141. package/sync/handler/converter/ScanCommandConverter.js +12 -11
  142. package/sync/handler/converter/ScanCommandConverter.js.map +1 -1
  143. package/sync/handler/converter/UpdateCommandConverter.js +12 -11
  144. package/sync/handler/converter/UpdateCommandConverter.js.map +1 -1
  145. package/sync/handler/converter/commands/BatchWriteCommandValue.js +34 -37
  146. package/sync/handler/converter/commands/BatchWriteCommandValue.js.map +1 -1
  147. package/sync/handler/converter/commands/DeleteCommandValue.js +19 -16
  148. package/sync/handler/converter/commands/DeleteCommandValue.js.map +1 -1
  149. package/sync/handler/converter/commands/NullCommandValue.js +8 -8
  150. package/sync/handler/converter/commands/NullCommandValue.js.map +1 -1
  151. package/sync/handler/converter/commands/PutCommandValue.js +19 -16
  152. package/sync/handler/converter/commands/PutCommandValue.js.map +1 -1
  153. package/sync/handler/converter/commands/UpdateCommandValue.js +19 -16
  154. package/sync/handler/converter/commands/UpdateCommandValue.js.map +1 -1
  155. package/sync/handler/types.js +0 -3
  156. package/sync/plugins/FilterOutRecordPlugin.js +17 -19
  157. package/sync/plugins/FilterOutRecordPlugin.js.map +1 -1
  158. package/sync/requestPlugin.js +33 -48
  159. package/sync/requestPlugin.js.map +1 -1
  160. package/sync/types.js +0 -3
  161. package/sync/utils/getTableType.js +10 -9
  162. package/sync/utils/getTableType.js.map +1 -1
  163. package/sync/utils/manifest.js +28 -35
  164. package/sync/utils/manifest.js.map +1 -1
  165. package/sync/utils/validateSystemInput.js +21 -29
  166. package/sync/utils/validateSystemInput.js.map +1 -1
  167. package/types.js +6 -5
  168. package/types.js.map +1 -1
  169. package/utils/createSystemName.js +5 -3
  170. package/utils/createSystemName.js.map +1 -1
  171. package/worker/actions/copyFile/CopyFile.js +183 -258
  172. package/worker/actions/copyFile/CopyFile.js.map +1 -1
  173. package/worker/actions/copyFile/copyFileAction.js +26 -34
  174. package/worker/actions/copyFile/copyFileAction.js.map +1 -1
  175. package/worker/actions/copyFile/copyFileSchema.js +13 -14
  176. package/worker/actions/copyFile/copyFileSchema.js.map +1 -1
  177. package/worker/actions/copyFile/types.js +0 -3
  178. package/worker/actions/createUser/CreateUser.js +51 -74
  179. package/worker/actions/createUser/CreateUser.js.map +1 -1
  180. package/worker/actions/createUser/createUserAction.js +24 -28
  181. package/worker/actions/createUser/createUserAction.js.map +1 -1
  182. package/worker/actions/createUser/createUserSchema.js +13 -14
  183. package/worker/actions/createUser/createUserSchema.js.map +1 -1
  184. package/worker/actions/createUser/types.js +0 -3
  185. package/worker/actions/deleteFile/DeleteFile.js +52 -67
  186. package/worker/actions/deleteFile/DeleteFile.js.map +1 -1
  187. package/worker/actions/deleteFile/deleteFileAction.js +21 -29
  188. package/worker/actions/deleteFile/deleteFileAction.js.map +1 -1
  189. package/worker/actions/deleteFile/deleteFileSchema.js +13 -14
  190. package/worker/actions/deleteFile/deleteFileSchema.js.map +1 -1
  191. package/worker/actions/deleteFile/types.js +0 -3
  192. package/worker/actions/deleteUser/DeleteUser.js +38 -52
  193. package/worker/actions/deleteUser/DeleteUser.js.map +1 -1
  194. package/worker/actions/deleteUser/deleteUserAction.js +22 -26
  195. package/worker/actions/deleteUser/deleteUserAction.js.map +1 -1
  196. package/worker/actions/deleteUser/deleteUserSchema.js +12 -13
  197. package/worker/actions/deleteUser/deleteUserSchema.js.map +1 -1
  198. package/worker/actions/deleteUser/types.js +0 -3
  199. package/worker/actions/logValidationError.js +5 -6
  200. package/worker/actions/logValidationError.js.map +1 -1
  201. package/worker/actions/removeCognitoUserAttributes.js +9 -15
  202. package/worker/actions/removeCognitoUserAttributes.js.map +1 -1
  203. package/worker/actions/updateUser/UpdateUser.js +48 -65
  204. package/worker/actions/updateUser/UpdateUser.js.map +1 -1
  205. package/worker/actions/updateUser/types.js +0 -3
  206. package/worker/actions/updateUser/updateUserAction.js +24 -28
  207. package/worker/actions/updateUser/updateUserAction.js.map +1 -1
  208. package/worker/actions/updateUser/updateUserSchema.js +13 -14
  209. package/worker/actions/updateUser/updateUserSchema.js.map +1 -1
  210. package/worker/createWorkerHandler.js +24 -30
  211. package/worker/createWorkerHandler.js.map +1 -1
  212. package/worker/handler/WorkerActionHandler.js +19 -21
  213. package/worker/handler/WorkerActionHandler.js.map +1 -1
  214. package/worker/handler/eventHandler.js +14 -16
  215. package/worker/handler/eventHandler.js.map +1 -1
  216. package/worker/plugins/WorkerActionPlugin.js +17 -16
  217. package/worker/plugins/WorkerActionPlugin.js.map +1 -1
  218. package/worker/types.js +0 -3
  219. package/index.js.map +0 -1
  220. package/resolver/app/abstractions/RecordHandler.js.map +0 -1
  221. package/resolver/app/abstractions/RecordsValidation.js.map +0 -1
  222. package/resolver/app/abstractions/ResolverApplication.js.map +0 -1
  223. package/resolver/app/abstractions/ResolverRecord.js.map +0 -1
  224. package/resolver/app/bundler/types.js.map +0 -1
  225. package/resolver/app/data/types.js.map +0 -1
  226. package/resolver/app/fetcher/types.js.map +0 -1
  227. package/resolver/app/ingestor/types.js.map +0 -1
  228. package/resolver/app/storer/types.js.map +0 -1
  229. package/resolver/deployment/types.js.map +0 -1
  230. package/resolver/recordTypes/fileManager/types.js.map +0 -1
  231. package/resolver/recordTypes/users/types.js.map +0 -1
  232. package/sync/handler/types.js.map +0 -1
  233. package/sync/types.js.map +0 -1
  234. package/worker/actions/copyFile/types.js.map +0 -1
  235. package/worker/actions/createUser/types.js.map +0 -1
  236. package/worker/actions/deleteFile/types.js.map +0 -1
  237. package/worker/actions/deleteUser/types.js.map +0 -1
  238. package/worker/actions/updateUser/types.js.map +0 -1
  239. package/worker/types.js.map +0 -1
@@ -1,123 +1,98 @@
1
1
  import { BatchGetCommand } from "@webiny/aws-sdk/client-dynamodb/index.js";
2
- import lodashChunk from "lodash/chunk.js";
2
+ import chunk from "lodash/chunk.js";
3
3
  import { SourceDataContainer } from "../data/SourceDataContainer.js";
4
4
  import { convertException } from "@webiny/utils";
5
5
  import { createRetry } from "../utils/Retry.js";
6
- export class Fetcher {
7
- constructor(params) {
8
- this.createDocumentClient = params.createDocumentClient;
9
- this.maxRetries = params.maxRetries || 10;
10
- this.retryDelay = params.retryDelay || 1000;
11
- }
12
- async exec(params) {
13
- const {
14
- deployment,
15
- items: input,
16
- table,
17
- maxBatchSize = 25
18
- } = params;
19
- if (input.length === 0) {
20
- return {
21
- items: SourceDataContainer.create()
22
- };
6
+ class Fetcher {
7
+ constructor(params){
8
+ this.createDocumentClient = params.createDocumentClient;
9
+ this.maxRetries = params.maxRetries || 10;
10
+ this.retryDelay = params.retryDelay || 1000;
23
11
  }
24
- const client = this.createDocumentClient(deployment);
25
- const results = await this.execute({
26
- client,
27
- table,
28
- maxBatchSize,
29
- records: input
30
- });
31
- const items = SourceDataContainer.create();
32
- for (const item of input) {
33
- const data = results.find(result => {
34
- return item.PK === result.PK && item.SK === result.SK;
35
- });
36
- items.add({
37
- PK: item.PK,
38
- SK: item.SK,
39
- table,
40
- source: deployment
41
- }, data || null);
12
+ async exec(params) {
13
+ const { deployment, items: input, table, maxBatchSize = 25 } = params;
14
+ if (0 === input.length) return {
15
+ items: SourceDataContainer.create()
16
+ };
17
+ const client = this.createDocumentClient(deployment);
18
+ const results = await this.execute({
19
+ client,
20
+ table,
21
+ maxBatchSize,
22
+ records: input
23
+ });
24
+ const items = SourceDataContainer.create();
25
+ for (const item of input){
26
+ const data = results.find((result)=>item.PK === result.PK && item.SK === result.SK);
27
+ items.add({
28
+ PK: item.PK,
29
+ SK: item.SK,
30
+ table,
31
+ source: deployment
32
+ }, data || null);
33
+ }
34
+ return {
35
+ items
36
+ };
42
37
  }
43
- return {
44
- items
45
- };
46
- }
47
- async execute(params) {
48
- const {
49
- client,
50
- table,
51
- records: items,
52
- maxBatchSize
53
- } = params;
54
- const batches = lodashChunk(items, maxBatchSize);
55
- const results = [];
56
- for (const batch of batches) {
57
- let keys = this.getKeys(batch);
58
- while (keys.length > 0) {
59
- const command = new BatchGetCommand({
60
- RequestItems: {
61
- [table.name]: {
62
- Keys: keys
38
+ async execute(params) {
39
+ const { client, table, records: items, maxBatchSize } = params;
40
+ const batches = chunk(items, maxBatchSize);
41
+ const results = [];
42
+ for (const batch of batches){
43
+ let keys = this.getKeys(batch);
44
+ while(keys.length > 0){
45
+ const command = new BatchGetCommand({
46
+ RequestItems: {
47
+ [table.name]: {
48
+ Keys: keys
49
+ }
50
+ }
51
+ });
52
+ const { items, unprocessedKeys } = await this.runCommand({
53
+ command,
54
+ client,
55
+ table: table.name
56
+ });
57
+ results.push(...items);
58
+ keys = unprocessedKeys;
63
59
  }
64
- }
60
+ }
61
+ return results;
62
+ }
63
+ async runCommand(params) {
64
+ const { command, table, client } = params;
65
+ const retry = createRetry({
66
+ maxRetries: this.maxRetries,
67
+ retryDelay: this.retryDelay
65
68
  });
66
- const {
67
- items,
68
- unprocessedKeys
69
- } = await this.runCommand({
70
- command,
71
- client,
72
- table: table.name
69
+ return await retry.retry(async ()=>{
70
+ const result = await client.send(command);
71
+ return {
72
+ items: result.Responses?.[table] || [],
73
+ unprocessedKeys: result.UnprocessedKeys?.[table]?.Keys || []
74
+ };
75
+ }, {
76
+ onFail: (ex)=>{
77
+ console.error(`Max retries reached. Could not fetch items from table: ${table}`);
78
+ console.log(convertException(ex));
79
+ console.log(JSON.stringify(command));
80
+ }
73
81
  });
74
- results.push(...items);
75
- keys = unprocessedKeys;
76
- }
77
82
  }
78
- return results;
79
- }
80
- async runCommand(params) {
81
- const {
82
- command,
83
- table,
84
- client
85
- } = params;
86
- const retry = createRetry({
87
- maxRetries: this.maxRetries,
88
- retryDelay: this.retryDelay
89
- });
90
- return await retry.retry(async () => {
91
- const result = await client.send(command);
92
- return {
93
- items: result.Responses?.[table] || [],
94
- unprocessedKeys: result.UnprocessedKeys?.[table]?.Keys || []
95
- };
96
- }, {
97
- onFail: ex => {
98
- console.error(`Max retries reached. Could not fetch items from table: ${table}`);
99
- console.log(convertException(ex));
100
- console.log(JSON.stringify(command));
101
- }
102
- });
103
- }
104
- getKeys(items) {
105
- const keys = new Map();
106
- for (const item of items) {
107
- const key = `PK:${item.PK}#SK:${item.SK}`;
108
- if (keys.has(key)) {
109
- continue;
110
- }
111
- keys.set(key, {
112
- PK: item.PK,
113
- SK: item.SK
114
- });
83
+ getKeys(items) {
84
+ const keys = new Map();
85
+ for (const item of items){
86
+ const key = `PK:${item.PK}#SK:${item.SK}`;
87
+ if (!keys.has(key)) keys.set(key, {
88
+ PK: item.PK,
89
+ SK: item.SK
90
+ });
91
+ }
92
+ return Array.from(keys.values());
115
93
  }
116
- return Array.from(keys.values());
117
- }
118
94
  }
119
- export const createFetcher = params => {
120
- return new Fetcher(params);
121
- };
95
+ const createFetcher = (params)=>new Fetcher(params);
96
+ export { Fetcher, createFetcher };
122
97
 
123
98
  //# sourceMappingURL=Fetcher.js.map
@@ -1 +1 @@
1
- {"version":3,"names":["BatchGetCommand","lodashChunk","SourceDataContainer","convertException","createRetry","Fetcher","constructor","params","createDocumentClient","maxRetries","retryDelay","exec","deployment","items","input","table","maxBatchSize","length","create","client","results","execute","records","item","data","find","result","PK","SK","add","source","batches","batch","keys","getKeys","command","RequestItems","name","Keys","unprocessedKeys","runCommand","push","retry","send","Responses","UnprocessedKeys","onFail","ex","console","error","log","JSON","stringify","Map","key","has","set","Array","from","values","createFetcher"],"sources":["Fetcher.ts"],"sourcesContent":["import type { IFetcher, IFetcherExecParams, IFetcherExecResult } from \"./types.js\";\nimport type { DynamoDBDocument } from \"@webiny/aws-sdk/client-dynamodb/index.js\";\nimport { BatchGetCommand } from \"@webiny/aws-sdk/client-dynamodb/index.js\";\nimport lodashChunk from \"lodash/chunk.js\";\nimport type { IDeployment } from \"~/resolver/deployment/types.js\";\nimport { SourceDataContainer } from \"~/resolver/app/data/SourceDataContainer.js\";\nimport type { GenericRecord } from \"@webiny/api/types.js\";\nimport { convertException } from \"@webiny/utils\";\nimport type { IStoreItem } from \"~/resolver/app/storer/types.js\";\nimport type { ITable } from \"~/sync/types.js\";\nimport { createRetry } from \"~/resolver/app/utils/Retry.js\";\n\nexport interface IFetcherParamsCreateDocumentClientCallable {\n (deployment: Pick<IDeployment, \"region\">): Pick<DynamoDBDocument, \"send\">;\n}\n\nexport interface IFetcherParams {\n maxRetries?: number;\n retryDelay?: number;\n createDocumentClient: IFetcherParamsCreateDocumentClientCallable;\n}\n\ninterface IKeys {\n PK: string;\n SK: string;\n}\n\ninterface IFetcherExecuteRunCommandParams {\n command: BatchGetCommand;\n table: string;\n client: Pick<DynamoDBDocument, \"send\">;\n}\n\ninterface IFetcherExecuteRunCommandResult<T = GenericRecord> {\n items: T[];\n unprocessedKeys: IKeys[];\n}\n\ninterface IFetchExecuteExecuteParamsItem {\n PK: string;\n SK: string;\n}\n\ninterface IFetchExecuteExecuteParams {\n maxBatchSize: number;\n client: Pick<DynamoDBDocument, \"send\">;\n table: ITable;\n records: IFetchExecuteExecuteParamsItem[];\n}\n\nexport class Fetcher implements IFetcher {\n private readonly createDocumentClient: IFetcherParamsCreateDocumentClientCallable;\n private readonly maxRetries: number;\n private readonly retryDelay: number;\n\n public constructor(params: IFetcherParams) {\n this.createDocumentClient = params.createDocumentClient;\n this.maxRetries = params.maxRetries || 10;\n this.retryDelay = params.retryDelay || 1000;\n }\n\n public async exec(params: IFetcherExecParams): Promise<IFetcherExecResult> {\n const { deployment, items: input, table, maxBatchSize = 25 } = params;\n if (input.length === 0) {\n return {\n items: SourceDataContainer.create()\n };\n }\n\n const client = this.createDocumentClient(deployment);\n\n const results = await this.execute({\n client,\n table,\n maxBatchSize,\n records: input\n });\n\n const items = SourceDataContainer.create();\n\n for (const item of input) {\n const data = results.find(result => {\n return item.PK === result.PK && item.SK === result.SK;\n });\n items.add(\n {\n PK: item.PK,\n SK: item.SK,\n table,\n source: deployment\n },\n data || null\n );\n }\n\n return {\n items\n };\n }\n\n private async execute<T = IStoreItem>(params: IFetchExecuteExecuteParams) {\n const { client, table, records: items, maxBatchSize } = params;\n const batches = lodashChunk(items, maxBatchSize);\n\n const results: T[] = [];\n for (const batch of batches) {\n let keys = this.getKeys(batch);\n\n while (keys.length > 0) {\n const command = new BatchGetCommand({\n RequestItems: {\n [table.name]: {\n Keys: keys\n }\n }\n });\n const { items, unprocessedKeys } = await this.runCommand<T>({\n command,\n client,\n table: table.name\n });\n\n results.push(...items);\n\n keys = unprocessedKeys;\n }\n }\n return results;\n }\n\n private async runCommand<T = GenericRecord>(\n params: IFetcherExecuteRunCommandParams\n ): Promise<IFetcherExecuteRunCommandResult<T>> {\n const { command, table, client } = params;\n\n const retry = createRetry({\n maxRetries: this.maxRetries,\n retryDelay: this.retryDelay\n });\n\n return await retry.retry(\n async () => {\n const result = await client.send(command);\n\n return {\n items: (result.Responses?.[table] || []) as T[],\n unprocessedKeys: (result.UnprocessedKeys?.[table]?.Keys || []) as IKeys[]\n };\n },\n {\n onFail: ex => {\n console.error(\n `Max retries reached. Could not fetch items from table: ${table}`\n );\n console.log(convertException(ex));\n console.log(JSON.stringify(command));\n }\n }\n );\n }\n\n private getKeys(items: IFetchExecuteExecuteParamsItem[]): IKeys[] {\n const keys = new Map<string, IKeys>();\n\n for (const item of items) {\n const key = `PK:${item.PK}#SK:${item.SK}`;\n if (keys.has(key)) {\n continue;\n }\n keys.set(key, {\n PK: item.PK,\n SK: item.SK\n });\n }\n\n return Array.from(keys.values());\n }\n}\n\nexport const createFetcher = (params: IFetcherParams): IFetcher => {\n return new Fetcher(params);\n};\n"],"mappings":"AAEA,SAASA,eAAe,QAAQ,0CAA0C;AAC1E,OAAOC,WAAW,MAAM,iBAAiB;AAEzC,SAASC,mBAAmB;AAE5B,SAASC,gBAAgB,QAAQ,eAAe;AAGhD,SAASC,WAAW;AAwCpB,OAAO,MAAMC,OAAO,CAAqB;EAK9BC,WAAWA,CAACC,MAAsB,EAAE;IACvC,IAAI,CAACC,oBAAoB,GAAGD,MAAM,CAACC,oBAAoB;IACvD,IAAI,CAACC,UAAU,GAAGF,MAAM,CAACE,UAAU,IAAI,EAAE;IACzC,IAAI,CAACC,UAAU,GAAGH,MAAM,CAACG,UAAU,IAAI,IAAI;EAC/C;EAEA,MAAaC,IAAIA,CAACJ,MAA0B,EAA+B;IACvE,MAAM;MAAEK,UAAU;MAAEC,KAAK,EAAEC,KAAK;MAAEC,KAAK;MAAEC,YAAY,GAAG;IAAG,CAAC,GAAGT,MAAM;IACrE,IAAIO,KAAK,CAACG,MAAM,KAAK,CAAC,EAAE;MACpB,OAAO;QACHJ,KAAK,EAAEX,mBAAmB,CAACgB,MAAM,CAAC;MACtC,CAAC;IACL;IAEA,MAAMC,MAAM,GAAG,IAAI,CAACX,oBAAoB,CAACI,UAAU,CAAC;IAEpD,MAAMQ,OAAO,GAAG,MAAM,IAAI,CAACC,OAAO,CAAC;MAC/BF,MAAM;MACNJ,KAAK;MACLC,YAAY;MACZM,OAAO,EAAER;IACb,CAAC,CAAC;IAEF,MAAMD,KAAK,GAAGX,mBAAmB,CAACgB,MAAM,CAAC,CAAC;IAE1C,KAAK,MAAMK,IAAI,IAAIT,KAAK,EAAE;MACtB,MAAMU,IAAI,GAAGJ,OAAO,CAACK,IAAI,CAACC,MAAM,IAAI;QAChC,OAAOH,IAAI,CAACI,EAAE,KAAKD,MAAM,CAACC,EAAE,IAAIJ,IAAI,CAACK,EAAE,KAAKF,MAAM,CAACE,EAAE;MACzD,CAAC,CAAC;MACFf,KAAK,CAACgB,GAAG,CACL;QACIF,EAAE,EAAEJ,IAAI,CAACI,EAAE;QACXC,EAAE,EAAEL,IAAI,CAACK,EAAE;QACXb,KAAK;QACLe,MAAM,EAAElB;MACZ,CAAC,EACDY,IAAI,IAAI,IACZ,CAAC;IACL;IAEA,OAAO;MACHX;IACJ,CAAC;EACL;EAEA,MAAcQ,OAAOA,CAAiBd,MAAkC,EAAE;IACtE,MAAM;MAAEY,MAAM;MAAEJ,KAAK;MAAEO,OAAO,EAAET,KAAK;MAAEG;IAAa,CAAC,GAAGT,MAAM;IAC9D,MAAMwB,OAAO,GAAG9B,WAAW,CAACY,KAAK,EAAEG,YAAY,CAAC;IAEhD,MAAMI,OAAY,GAAG,EAAE;IACvB,KAAK,MAAMY,KAAK,IAAID,OAAO,EAAE;MACzB,IAAIE,IAAI,GAAG,IAAI,CAACC,OAAO,CAACF,KAAK,CAAC;MAE9B,OAAOC,IAAI,CAAChB,MAAM,GAAG,CAAC,EAAE;QACpB,MAAMkB,OAAO,GAAG,IAAInC,eAAe,CAAC;UAChCoC,YAAY,EAAE;YACV,CAACrB,KAAK,CAACsB,IAAI,GAAG;cACVC,IAAI,EAAEL;YACV;UACJ;QACJ,CAAC,CAAC;QACF,MAAM;UAAEpB,KAAK;UAAE0B;QAAgB,CAAC,GAAG,MAAM,IAAI,CAACC,UAAU,CAAI;UACxDL,OAAO;UACPhB,MAAM;UACNJ,KAAK,EAAEA,KAAK,CAACsB;QACjB,CAAC,CAAC;QAEFjB,OAAO,CAACqB,IAAI,CAAC,GAAG5B,KAAK,CAAC;QAEtBoB,IAAI,GAAGM,eAAe;MAC1B;IACJ;IACA,OAAOnB,OAAO;EAClB;EAEA,MAAcoB,UAAUA,CACpBjC,MAAuC,EACI;IAC3C,MAAM;MAAE4B,OAAO;MAAEpB,KAAK;MAAEI;IAAO,CAAC,GAAGZ,MAAM;IAEzC,MAAMmC,KAAK,GAAGtC,WAAW,CAAC;MACtBK,UAAU,EAAE,IAAI,CAACA,UAAU;MAC3BC,UAAU,EAAE,IAAI,CAACA;IACrB,CAAC,CAAC;IAEF,OAAO,MAAMgC,KAAK,CAACA,KAAK,CACpB,YAAY;MACR,MAAMhB,MAAM,GAAG,MAAMP,MAAM,CAACwB,IAAI,CAACR,OAAO,CAAC;MAEzC,OAAO;QACHtB,KAAK,EAAGa,MAAM,CAACkB,SAAS,GAAG7B,KAAK,CAAC,IAAI,EAAU;QAC/CwB,eAAe,EAAGb,MAAM,CAACmB,eAAe,GAAG9B,KAAK,CAAC,EAAEuB,IAAI,IAAI;MAC/D,CAAC;IACL,CAAC,EACD;MACIQ,MAAM,EAAEC,EAAE,IAAI;QACVC,OAAO,CAACC,KAAK,CACT,0DAA0DlC,KAAK,EACnE,CAAC;QACDiC,OAAO,CAACE,GAAG,CAAC/C,gBAAgB,CAAC4C,EAAE,CAAC,CAAC;QACjCC,OAAO,CAACE,GAAG,CAACC,IAAI,CAACC,SAAS,CAACjB,OAAO,CAAC,CAAC;MACxC;IACJ,CACJ,CAAC;EACL;EAEQD,OAAOA,CAACrB,KAAuC,EAAW;IAC9D,MAAMoB,IAAI,GAAG,IAAIoB,GAAG,CAAgB,CAAC;IAErC,KAAK,MAAM9B,IAAI,IAAIV,KAAK,EAAE;MACtB,MAAMyC,GAAG,GAAG,MAAM/B,IAAI,CAACI,EAAE,OAAOJ,IAAI,CAACK,EAAE,EAAE;MACzC,IAAIK,IAAI,CAACsB,GAAG,CAACD,GAAG,CAAC,EAAE;QACf;MACJ;MACArB,IAAI,CAACuB,GAAG,CAACF,GAAG,EAAE;QACV3B,EAAE,EAAEJ,IAAI,CAACI,EAAE;QACXC,EAAE,EAAEL,IAAI,CAACK;MACb,CAAC,CAAC;IACN;IAEA,OAAO6B,KAAK,CAACC,IAAI,CAACzB,IAAI,CAAC0B,MAAM,CAAC,CAAC,CAAC;EACpC;AACJ;AAEA,OAAO,MAAMC,aAAa,GAAIrD,MAAsB,IAAe;EAC/D,OAAO,IAAIF,OAAO,CAACE,MAAM,CAAC;AAC9B,CAAC","ignoreList":[]}
1
+ {"version":3,"file":"resolver/app/fetcher/Fetcher.js","sources":["../../../../src/resolver/app/fetcher/Fetcher.ts"],"sourcesContent":["import type { IFetcher, IFetcherExecParams, IFetcherExecResult } from \"./types.js\";\nimport type { DynamoDBDocument } from \"@webiny/aws-sdk/client-dynamodb/index.js\";\nimport { BatchGetCommand } from \"@webiny/aws-sdk/client-dynamodb/index.js\";\nimport lodashChunk from \"lodash/chunk.js\";\nimport type { IDeployment } from \"~/resolver/deployment/types.js\";\nimport { SourceDataContainer } from \"~/resolver/app/data/SourceDataContainer.js\";\nimport type { GenericRecord } from \"@webiny/api/types.js\";\nimport { convertException } from \"@webiny/utils\";\nimport type { IStoreItem } from \"~/resolver/app/storer/types.js\";\nimport type { ITable } from \"~/sync/types.js\";\nimport { createRetry } from \"~/resolver/app/utils/Retry.js\";\n\nexport interface IFetcherParamsCreateDocumentClientCallable {\n (deployment: Pick<IDeployment, \"region\">): Pick<DynamoDBDocument, \"send\">;\n}\n\nexport interface IFetcherParams {\n maxRetries?: number;\n retryDelay?: number;\n createDocumentClient: IFetcherParamsCreateDocumentClientCallable;\n}\n\ninterface IKeys {\n PK: string;\n SK: string;\n}\n\ninterface IFetcherExecuteRunCommandParams {\n command: BatchGetCommand;\n table: string;\n client: Pick<DynamoDBDocument, \"send\">;\n}\n\ninterface IFetcherExecuteRunCommandResult<T = GenericRecord> {\n items: T[];\n unprocessedKeys: IKeys[];\n}\n\ninterface IFetchExecuteExecuteParamsItem {\n PK: string;\n SK: string;\n}\n\ninterface IFetchExecuteExecuteParams {\n maxBatchSize: number;\n client: Pick<DynamoDBDocument, \"send\">;\n table: ITable;\n records: IFetchExecuteExecuteParamsItem[];\n}\n\nexport class Fetcher implements IFetcher {\n private readonly createDocumentClient: IFetcherParamsCreateDocumentClientCallable;\n private readonly maxRetries: number;\n private readonly retryDelay: number;\n\n public constructor(params: IFetcherParams) {\n this.createDocumentClient = params.createDocumentClient;\n this.maxRetries = params.maxRetries || 10;\n this.retryDelay = params.retryDelay || 1000;\n }\n\n public async exec(params: IFetcherExecParams): Promise<IFetcherExecResult> {\n const { deployment, items: input, table, maxBatchSize = 25 } = params;\n if (input.length === 0) {\n return {\n items: SourceDataContainer.create()\n };\n }\n\n const client = this.createDocumentClient(deployment);\n\n const results = await this.execute({\n client,\n table,\n maxBatchSize,\n records: input\n });\n\n const items = SourceDataContainer.create();\n\n for (const item of input) {\n const data = results.find(result => {\n return item.PK === result.PK && item.SK === result.SK;\n });\n items.add(\n {\n PK: item.PK,\n SK: item.SK,\n table,\n source: deployment\n },\n data || null\n );\n }\n\n return {\n items\n };\n }\n\n private async execute<T = IStoreItem>(params: IFetchExecuteExecuteParams) {\n const { client, table, records: items, maxBatchSize } = params;\n const batches = lodashChunk(items, maxBatchSize);\n\n const results: T[] = [];\n for (const batch of batches) {\n let keys = this.getKeys(batch);\n\n while (keys.length > 0) {\n const command = new BatchGetCommand({\n RequestItems: {\n [table.name]: {\n Keys: keys\n }\n }\n });\n const { items, unprocessedKeys } = await this.runCommand<T>({\n command,\n client,\n table: table.name\n });\n\n results.push(...items);\n\n keys = unprocessedKeys;\n }\n }\n return results;\n }\n\n private async runCommand<T = GenericRecord>(\n params: IFetcherExecuteRunCommandParams\n ): Promise<IFetcherExecuteRunCommandResult<T>> {\n const { command, table, client } = params;\n\n const retry = createRetry({\n maxRetries: this.maxRetries,\n retryDelay: this.retryDelay\n });\n\n return await retry.retry(\n async () => {\n const result = await client.send(command);\n\n return {\n items: (result.Responses?.[table] || []) as T[],\n unprocessedKeys: (result.UnprocessedKeys?.[table]?.Keys || []) as IKeys[]\n };\n },\n {\n onFail: ex => {\n console.error(\n `Max retries reached. Could not fetch items from table: ${table}`\n );\n console.log(convertException(ex));\n console.log(JSON.stringify(command));\n }\n }\n );\n }\n\n private getKeys(items: IFetchExecuteExecuteParamsItem[]): IKeys[] {\n const keys = new Map<string, IKeys>();\n\n for (const item of items) {\n const key = `PK:${item.PK}#SK:${item.SK}`;\n if (keys.has(key)) {\n continue;\n }\n keys.set(key, {\n PK: item.PK,\n SK: item.SK\n });\n }\n\n return Array.from(keys.values());\n }\n}\n\nexport const createFetcher = (params: IFetcherParams): IFetcher => {\n return new Fetcher(params);\n};\n"],"names":["Fetcher","params","deployment","input","table","maxBatchSize","SourceDataContainer","client","results","items","item","data","result","batches","lodashChunk","batch","keys","command","BatchGetCommand","unprocessedKeys","retry","createRetry","ex","console","convertException","JSON","Map","key","Array","createFetcher"],"mappings":";;;;;AAkDO,MAAMA;IAKT,YAAmBC,MAAsB,CAAE;QACvC,IAAI,CAAC,oBAAoB,GAAGA,OAAO,oBAAoB;QACvD,IAAI,CAAC,UAAU,GAAGA,OAAO,UAAU,IAAI;QACvC,IAAI,CAAC,UAAU,GAAGA,OAAO,UAAU,IAAI;IAC3C;IAEA,MAAa,KAAKA,MAA0B,EAA+B;QACvE,MAAM,EAAEC,UAAU,EAAE,OAAOC,KAAK,EAAEC,KAAK,EAAEC,eAAe,EAAE,EAAE,GAAGJ;QAC/D,IAAIE,AAAiB,MAAjBA,MAAM,MAAM,EACZ,OAAO;YACH,OAAOG,oBAAoB,MAAM;QACrC;QAGJ,MAAMC,SAAS,IAAI,CAAC,oBAAoB,CAACL;QAEzC,MAAMM,UAAU,MAAM,IAAI,CAAC,OAAO,CAAC;YAC/BD;YACAH;YACAC;YACA,SAASF;QACb;QAEA,MAAMM,QAAQH,oBAAoB,MAAM;QAExC,KAAK,MAAMI,QAAQP,MAAO;YACtB,MAAMQ,OAAOH,QAAQ,IAAI,CAACI,CAAAA,SACfF,KAAK,EAAE,KAAKE,OAAO,EAAE,IAAIF,KAAK,EAAE,KAAKE,OAAO,EAAE;YAEzDH,MAAM,GAAG,CACL;gBACI,IAAIC,KAAK,EAAE;gBACX,IAAIA,KAAK,EAAE;gBACXN;gBACA,QAAQF;YACZ,GACAS,QAAQ;QAEhB;QAEA,OAAO;YACHF;QACJ;IACJ;IAEA,MAAc,QAAwBR,MAAkC,EAAE;QACtE,MAAM,EAAEM,MAAM,EAAEH,KAAK,EAAE,SAASK,KAAK,EAAEJ,YAAY,EAAE,GAAGJ;QACxD,MAAMY,UAAUC,MAAYL,OAAOJ;QAEnC,MAAMG,UAAe,EAAE;QACvB,KAAK,MAAMO,SAASF,QAAS;YACzB,IAAIG,OAAO,IAAI,CAAC,OAAO,CAACD;YAExB,MAAOC,KAAK,MAAM,GAAG,EAAG;gBACpB,MAAMC,UAAU,IAAIC,gBAAgB;oBAChC,cAAc;wBACV,CAACd,MAAM,IAAI,CAAC,EAAE;4BACV,MAAMY;wBACV;oBACJ;gBACJ;gBACA,MAAM,EAAEP,KAAK,EAAEU,eAAe,EAAE,GAAG,MAAM,IAAI,CAAC,UAAU,CAAI;oBACxDF;oBACAV;oBACA,OAAOH,MAAM,IAAI;gBACrB;gBAEAI,QAAQ,IAAI,IAAIC;gBAEhBO,OAAOG;YACX;QACJ;QACA,OAAOX;IACX;IAEA,MAAc,WACVP,MAAuC,EACI;QAC3C,MAAM,EAAEgB,OAAO,EAAEb,KAAK,EAAEG,MAAM,EAAE,GAAGN;QAEnC,MAAMmB,QAAQC,YAAY;YACtB,YAAY,IAAI,CAAC,UAAU;YAC3B,YAAY,IAAI,CAAC,UAAU;QAC/B;QAEA,OAAO,MAAMD,MAAM,KAAK,CACpB;YACI,MAAMR,SAAS,MAAML,OAAO,IAAI,CAACU;YAEjC,OAAO;gBACH,OAAQL,OAAO,SAAS,EAAE,CAACR,MAAM,IAAI,EAAE;gBACvC,iBAAkBQ,OAAO,eAAe,EAAE,CAACR,MAAM,EAAE,QAAQ,EAAE;YACjE;QACJ,GACA;YACI,QAAQkB,CAAAA;gBACJC,QAAQ,KAAK,CACT,CAAC,uDAAuD,EAAEnB,OAAO;gBAErEmB,QAAQ,GAAG,CAACC,iBAAiBF;gBAC7BC,QAAQ,GAAG,CAACE,KAAK,SAAS,CAACR;YAC/B;QACJ;IAER;IAEQ,QAAQR,KAAuC,EAAW;QAC9D,MAAMO,OAAO,IAAIU;QAEjB,KAAK,MAAMhB,QAAQD,MAAO;YACtB,MAAMkB,MAAM,CAAC,GAAG,EAAEjB,KAAK,EAAE,CAAC,IAAI,EAAEA,KAAK,EAAE,EAAE;YACzC,KAAIM,KAAK,GAAG,CAACW,MAGbX,KAAK,GAAG,CAACW,KAAK;gBACV,IAAIjB,KAAK,EAAE;gBACX,IAAIA,KAAK,EAAE;YACf;QACJ;QAEA,OAAOkB,MAAM,IAAI,CAACZ,KAAK,MAAM;IACjC;AACJ;AAEO,MAAMa,gBAAgB,CAAC5B,SACnB,IAAID,QAAQC"}
@@ -1,3 +0,0 @@
1
- export {};
2
-
3
- //# sourceMappingURL=types.js.map
@@ -1,36 +1,31 @@
1
- export class Ingestor {
2
- constructor(params) {
3
- this.createIngestorResult = params.createIngestorResult;
4
- this.getSource = params.getSource;
5
- }
6
- async ingest(params) {
7
- const {
8
- records
9
- } = params;
10
- const result = this.createIngestorResult();
11
- for (const record of records) {
12
- const detail = record.body.detail;
13
- const source = this.getSource(detail.source.name);
14
- if (!source) {
15
- console.error(`Could not find deployment for SQS Record source: ${detail.source.name}. More info in next log line.`);
16
- console.log(JSON.stringify({
17
- source: detail.source,
18
- items: detail.items
19
- }));
20
- continue;
21
- }
22
- for (const item of detail.items) {
23
- result.add({
24
- item,
25
- source
26
- });
27
- }
1
+ class Ingestor {
2
+ constructor(params){
3
+ this.createIngestorResult = params.createIngestorResult;
4
+ this.getSource = params.getSource;
5
+ }
6
+ async ingest(params) {
7
+ const { records } = params;
8
+ const result = this.createIngestorResult();
9
+ for (const record of records){
10
+ const detail = record.body.detail;
11
+ const source = this.getSource(detail.source.name);
12
+ if (!source) {
13
+ console.error(`Could not find deployment for SQS Record source: ${detail.source.name}. More info in next log line.`);
14
+ console.log(JSON.stringify({
15
+ source: detail.source,
16
+ items: detail.items
17
+ }));
18
+ continue;
19
+ }
20
+ for (const item of detail.items)result.add({
21
+ item,
22
+ source
23
+ });
24
+ }
25
+ return result;
28
26
  }
29
- return result;
30
- }
31
27
  }
32
- export const createIngestor = params => {
33
- return new Ingestor(params);
34
- };
28
+ const createIngestor = (params)=>new Ingestor(params);
29
+ export { Ingestor, createIngestor };
35
30
 
36
31
  //# sourceMappingURL=Ingestor.js.map
@@ -1 +1 @@
1
- {"version":3,"names":["Ingestor","constructor","params","createIngestorResult","getSource","ingest","records","result","record","detail","body","source","name","console","error","log","JSON","stringify","items","item","add","createIngestor"],"sources":["Ingestor.ts"],"sourcesContent":["import type { IIngestor, IIngestorIngestParams, IIngestorResult } from \"./types.js\";\nimport type { IDeployment } from \"~/resolver/deployment/types.js\";\n\nexport interface IIngestorParams {\n createIngestorResult: () => IIngestorResult;\n getSource: (name: string) => IDeployment | null;\n}\n\nexport class Ingestor implements IIngestor {\n private readonly createIngestorResult: () => IIngestorResult;\n private readonly getSource: (name: string) => IDeployment | null;\n\n public constructor(params: IIngestorParams) {\n this.createIngestorResult = params.createIngestorResult;\n this.getSource = params.getSource;\n }\n\n public async ingest(params: IIngestorIngestParams): Promise<IIngestorResult> {\n const { records } = params;\n\n const result = this.createIngestorResult();\n\n for (const record of records) {\n const detail = record.body.detail;\n const source = this.getSource(detail.source.name);\n if (!source) {\n console.error(\n `Could not find deployment for SQS Record source: ${detail.source.name}. More info in next log line.`\n );\n console.log(\n JSON.stringify({\n source: detail.source,\n items: detail.items\n })\n );\n continue;\n }\n\n for (const item of detail.items) {\n result.add({\n item,\n source\n });\n }\n }\n\n return result;\n }\n}\n\nexport const createIngestor = (params: IIngestorParams): IIngestor => {\n return new Ingestor(params);\n};\n"],"mappings":"AAQA,OAAO,MAAMA,QAAQ,CAAsB;EAIhCC,WAAWA,CAACC,MAAuB,EAAE;IACxC,IAAI,CAACC,oBAAoB,GAAGD,MAAM,CAACC,oBAAoB;IACvD,IAAI,CAACC,SAAS,GAAGF,MAAM,CAACE,SAAS;EACrC;EAEA,MAAaC,MAAMA,CAACH,MAA6B,EAA4B;IACzE,MAAM;MAAEI;IAAQ,CAAC,GAAGJ,MAAM;IAE1B,MAAMK,MAAM,GAAG,IAAI,CAACJ,oBAAoB,CAAC,CAAC;IAE1C,KAAK,MAAMK,MAAM,IAAIF,OAAO,EAAE;MAC1B,MAAMG,MAAM,GAAGD,MAAM,CAACE,IAAI,CAACD,MAAM;MACjC,MAAME,MAAM,GAAG,IAAI,CAACP,SAAS,CAACK,MAAM,CAACE,MAAM,CAACC,IAAI,CAAC;MACjD,IAAI,CAACD,MAAM,EAAE;QACTE,OAAO,CAACC,KAAK,CACT,oDAAoDL,MAAM,CAACE,MAAM,CAACC,IAAI,+BAC1E,CAAC;QACDC,OAAO,CAACE,GAAG,CACPC,IAAI,CAACC,SAAS,CAAC;UACXN,MAAM,EAAEF,MAAM,CAACE,MAAM;UACrBO,KAAK,EAAET,MAAM,CAACS;QAClB,CAAC,CACL,CAAC;QACD;MACJ;MAEA,KAAK,MAAMC,IAAI,IAAIV,MAAM,CAACS,KAAK,EAAE;QAC7BX,MAAM,CAACa,GAAG,CAAC;UACPD,IAAI;UACJR;QACJ,CAAC,CAAC;MACN;IACJ;IAEA,OAAOJ,MAAM;EACjB;AACJ;AAEA,OAAO,MAAMc,cAAc,GAAInB,MAAuB,IAAgB;EAClE,OAAO,IAAIF,QAAQ,CAACE,MAAM,CAAC;AAC/B,CAAC","ignoreList":[]}
1
+ {"version":3,"file":"resolver/app/ingestor/Ingestor.js","sources":["../../../../src/resolver/app/ingestor/Ingestor.ts"],"sourcesContent":["import type { IIngestor, IIngestorIngestParams, IIngestorResult } from \"./types.js\";\nimport type { IDeployment } from \"~/resolver/deployment/types.js\";\n\nexport interface IIngestorParams {\n createIngestorResult: () => IIngestorResult;\n getSource: (name: string) => IDeployment | null;\n}\n\nexport class Ingestor implements IIngestor {\n private readonly createIngestorResult: () => IIngestorResult;\n private readonly getSource: (name: string) => IDeployment | null;\n\n public constructor(params: IIngestorParams) {\n this.createIngestorResult = params.createIngestorResult;\n this.getSource = params.getSource;\n }\n\n public async ingest(params: IIngestorIngestParams): Promise<IIngestorResult> {\n const { records } = params;\n\n const result = this.createIngestorResult();\n\n for (const record of records) {\n const detail = record.body.detail;\n const source = this.getSource(detail.source.name);\n if (!source) {\n console.error(\n `Could not find deployment for SQS Record source: ${detail.source.name}. More info in next log line.`\n );\n console.log(\n JSON.stringify({\n source: detail.source,\n items: detail.items\n })\n );\n continue;\n }\n\n for (const item of detail.items) {\n result.add({\n item,\n source\n });\n }\n }\n\n return result;\n }\n}\n\nexport const createIngestor = (params: IIngestorParams): IIngestor => {\n return new Ingestor(params);\n};\n"],"names":["Ingestor","params","records","result","record","detail","source","console","JSON","item","createIngestor"],"mappings":"AAQO,MAAMA;IAIT,YAAmBC,MAAuB,CAAE;QACxC,IAAI,CAAC,oBAAoB,GAAGA,OAAO,oBAAoB;QACvD,IAAI,CAAC,SAAS,GAAGA,OAAO,SAAS;IACrC;IAEA,MAAa,OAAOA,MAA6B,EAA4B;QACzE,MAAM,EAAEC,OAAO,EAAE,GAAGD;QAEpB,MAAME,SAAS,IAAI,CAAC,oBAAoB;QAExC,KAAK,MAAMC,UAAUF,QAAS;YAC1B,MAAMG,SAASD,OAAO,IAAI,CAAC,MAAM;YACjC,MAAME,SAAS,IAAI,CAAC,SAAS,CAACD,OAAO,MAAM,CAAC,IAAI;YAChD,IAAI,CAACC,QAAQ;gBACTC,QAAQ,KAAK,CACT,CAAC,iDAAiD,EAAEF,OAAO,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC;gBAEzGE,QAAQ,GAAG,CACPC,KAAK,SAAS,CAAC;oBACX,QAAQH,OAAO,MAAM;oBACrB,OAAOA,OAAO,KAAK;gBACvB;gBAEJ;YACJ;YAEA,KAAK,MAAMI,QAAQJ,OAAO,KAAK,CAC3BF,OAAO,GAAG,CAAC;gBACPM;gBACAH;YACJ;QAER;QAEA,OAAOH;IACX;AACJ;AAEO,MAAMO,iBAAiB,CAACT,SACpB,IAAID,SAASC"}
@@ -1,37 +1,33 @@
1
- export class IngestorResult {
2
- items = new Map();
3
- add(params) {
4
- const {
5
- item,
6
- source
7
- } = params;
8
- let table;
9
- try {
10
- table = source.getTable(item.tableType);
11
- } catch {
12
- console.error(`Could not find table for SQS Record source: ${item.tableName} / ${item.tableType}. More info in next log line.`);
13
- console.log(JSON.stringify(item));
14
- return;
1
+ class IngestorResult {
2
+ add(params) {
3
+ const { item, source } = params;
4
+ let table;
5
+ try {
6
+ table = source.getTable(item.tableType);
7
+ } catch {
8
+ console.error(`Could not find table for SQS Record source: ${item.tableName} / ${item.tableType}. More info in next log line.`);
9
+ console.log(JSON.stringify(item));
10
+ return;
11
+ }
12
+ const command = "delete" === item.command ? "delete" : "put";
13
+ const key = `PK:${item.PK}#SK:${item.SK}#cmd:${command}#source:${source.name}`;
14
+ if (this.items.has(key)) return;
15
+ this.items.set(key, {
16
+ PK: item.PK,
17
+ SK: item.SK,
18
+ source,
19
+ command,
20
+ table
21
+ });
15
22
  }
16
- const command = item.command === "delete" ? "delete" : "put";
17
- const key = `PK:${item.PK}#SK:${item.SK}#cmd:${command}#source:${source.name}`;
18
- if (this.items.has(key)) {
19
- return;
23
+ getItems() {
24
+ return Array.from(this.items.values());
25
+ }
26
+ constructor(){
27
+ this.items = new Map();
20
28
  }
21
- this.items.set(key, {
22
- PK: item.PK,
23
- SK: item.SK,
24
- source,
25
- command,
26
- table
27
- });
28
- }
29
- getItems() {
30
- return Array.from(this.items.values());
31
- }
32
29
  }
33
- export const createIngestorResult = () => {
34
- return new IngestorResult();
35
- };
30
+ const createIngestorResult = ()=>new IngestorResult();
31
+ export { IngestorResult, createIngestorResult };
36
32
 
37
33
  //# sourceMappingURL=IngestorResult.js.map
@@ -1 +1 @@
1
- {"version":3,"names":["IngestorResult","items","Map","add","params","item","source","table","getTable","tableType","console","error","tableName","log","JSON","stringify","command","key","PK","SK","name","has","set","getItems","Array","from","values","createIngestorResult"],"sources":["IngestorResult.ts"],"sourcesContent":["import type { IIngestorResult, IIngestorResultAddParams, IIngestorResultItem } from \"./types.js\";\nimport type { ITable } from \"~/sync/types.js\";\n\nexport class IngestorResult implements IIngestorResult {\n private readonly items: Map<string, IIngestorResultItem> = new Map();\n\n public add(params: IIngestorResultAddParams): void {\n const { item, source } = params;\n let table: ITable;\n try {\n table = source.getTable(item.tableType);\n } catch {\n console.error(\n `Could not find table for SQS Record source: ${item.tableName} / ${item.tableType}. More info in next log line.`\n );\n console.log(JSON.stringify(item));\n return;\n }\n\n const command = item.command === \"delete\" ? \"delete\" : \"put\";\n\n const key = `PK:${item.PK}#SK:${item.SK}#cmd:${command}#source:${source.name}`;\n\n if (this.items.has(key)) {\n return;\n }\n\n this.items.set(key, {\n PK: item.PK,\n SK: item.SK,\n source,\n command,\n table\n });\n }\n\n public getItems(): IIngestorResultItem[] {\n return Array.from(this.items.values());\n }\n}\n\nexport const createIngestorResult = (): IIngestorResult => {\n return new IngestorResult();\n};\n"],"mappings":"AAGA,OAAO,MAAMA,cAAc,CAA4B;EAClCC,KAAK,GAAqC,IAAIC,GAAG,CAAC,CAAC;EAE7DC,GAAGA,CAACC,MAAgC,EAAQ;IAC/C,MAAM;MAAEC,IAAI;MAAEC;IAAO,CAAC,GAAGF,MAAM;IAC/B,IAAIG,KAAa;IACjB,IAAI;MACAA,KAAK,GAAGD,MAAM,CAACE,QAAQ,CAACH,IAAI,CAACI,SAAS,CAAC;IAC3C,CAAC,CAAC,MAAM;MACJC,OAAO,CAACC,KAAK,CACT,+CAA+CN,IAAI,CAACO,SAAS,MAAMP,IAAI,CAACI,SAAS,+BACrF,CAAC;MACDC,OAAO,CAACG,GAAG,CAACC,IAAI,CAACC,SAAS,CAACV,IAAI,CAAC,CAAC;MACjC;IACJ;IAEA,MAAMW,OAAO,GAAGX,IAAI,CAACW,OAAO,KAAK,QAAQ,GAAG,QAAQ,GAAG,KAAK;IAE5D,MAAMC,GAAG,GAAG,MAAMZ,IAAI,CAACa,EAAE,OAAOb,IAAI,CAACc,EAAE,QAAQH,OAAO,WAAWV,MAAM,CAACc,IAAI,EAAE;IAE9E,IAAI,IAAI,CAACnB,KAAK,CAACoB,GAAG,CAACJ,GAAG,CAAC,EAAE;MACrB;IACJ;IAEA,IAAI,CAAChB,KAAK,CAACqB,GAAG,CAACL,GAAG,EAAE;MAChBC,EAAE,EAAEb,IAAI,CAACa,EAAE;MACXC,EAAE,EAAEd,IAAI,CAACc,EAAE;MACXb,MAAM;MACNU,OAAO;MACPT;IACJ,CAAC,CAAC;EACN;EAEOgB,QAAQA,CAAA,EAA0B;IACrC,OAAOC,KAAK,CAACC,IAAI,CAAC,IAAI,CAACxB,KAAK,CAACyB,MAAM,CAAC,CAAC,CAAC;EAC1C;AACJ;AAEA,OAAO,MAAMC,oBAAoB,GAAGA,CAAA,KAAuB;EACvD,OAAO,IAAI3B,cAAc,CAAC,CAAC;AAC/B,CAAC","ignoreList":[]}
1
+ {"version":3,"file":"resolver/app/ingestor/IngestorResult.js","sources":["../../../../src/resolver/app/ingestor/IngestorResult.ts"],"sourcesContent":["import type { IIngestorResult, IIngestorResultAddParams, IIngestorResultItem } from \"./types.js\";\nimport type { ITable } from \"~/sync/types.js\";\n\nexport class IngestorResult implements IIngestorResult {\n private readonly items: Map<string, IIngestorResultItem> = new Map();\n\n public add(params: IIngestorResultAddParams): void {\n const { item, source } = params;\n let table: ITable;\n try {\n table = source.getTable(item.tableType);\n } catch {\n console.error(\n `Could not find table for SQS Record source: ${item.tableName} / ${item.tableType}. More info in next log line.`\n );\n console.log(JSON.stringify(item));\n return;\n }\n\n const command = item.command === \"delete\" ? \"delete\" : \"put\";\n\n const key = `PK:${item.PK}#SK:${item.SK}#cmd:${command}#source:${source.name}`;\n\n if (this.items.has(key)) {\n return;\n }\n\n this.items.set(key, {\n PK: item.PK,\n SK: item.SK,\n source,\n command,\n table\n });\n }\n\n public getItems(): IIngestorResultItem[] {\n return Array.from(this.items.values());\n }\n}\n\nexport const createIngestorResult = (): IIngestorResult => {\n return new IngestorResult();\n};\n"],"names":["IngestorResult","params","item","source","table","console","JSON","command","key","Array","Map","createIngestorResult"],"mappings":"AAGO,MAAMA;IAGF,IAAIC,MAAgC,EAAQ;QAC/C,MAAM,EAAEC,IAAI,EAAEC,MAAM,EAAE,GAAGF;QACzB,IAAIG;QACJ,IAAI;YACAA,QAAQD,OAAO,QAAQ,CAACD,KAAK,SAAS;QAC1C,EAAE,OAAM;YACJG,QAAQ,KAAK,CACT,CAAC,4CAA4C,EAAEH,KAAK,SAAS,CAAC,GAAG,EAAEA,KAAK,SAAS,CAAC,6BAA6B,CAAC;YAEpHG,QAAQ,GAAG,CAACC,KAAK,SAAS,CAACJ;YAC3B;QACJ;QAEA,MAAMK,UAAUL,AAAiB,aAAjBA,KAAK,OAAO,GAAgB,WAAW;QAEvD,MAAMM,MAAM,CAAC,GAAG,EAAEN,KAAK,EAAE,CAAC,IAAI,EAAEA,KAAK,EAAE,CAAC,KAAK,EAAEK,QAAQ,QAAQ,EAAEJ,OAAO,IAAI,EAAE;QAE9E,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAACK,MACf;QAGJ,IAAI,CAAC,KAAK,CAAC,GAAG,CAACA,KAAK;YAChB,IAAIN,KAAK,EAAE;YACX,IAAIA,KAAK,EAAE;YACXC;YACAI;YACAH;QACJ;IACJ;IAEO,WAAkC;QACrC,OAAOK,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM;IACvC;;aAlCiB,KAAK,GAAqC,IAAIC;;AAmCnE;AAEO,MAAMC,uBAAuB,IACzB,IAAIX"}
@@ -1,3 +0,0 @@
1
- export {};
2
-
3
- //# sourceMappingURL=types.js.map
@@ -1,95 +1,81 @@
1
1
  import { DeleteCommand, PutCommand } from "@webiny/aws-sdk/client-dynamodb/index.js";
2
2
  import { createRetry } from "../utils/Retry.js";
3
3
  import { convertException } from "@webiny/utils";
4
- export class Storer {
5
- constructor(params) {
6
- this.maxRetries = params.maxRetries || 10;
7
- this.retryDelay = params.retryDelay || 1000;
8
- this.createDocumentClient = params.createDocumentClient;
9
- this.afterEach = params.afterEach;
10
- this.onError = params.onError;
11
- }
12
- async store(params) {
13
- const {
14
- deployment,
15
- bundle,
16
- table,
17
- command,
18
- items
19
- } = params;
20
- if (items.length === 0) {
21
- return;
4
+ class Storer {
5
+ constructor(params){
6
+ this.maxRetries = params.maxRetries || 10;
7
+ this.retryDelay = params.retryDelay || 1000;
8
+ this.createDocumentClient = params.createDocumentClient;
9
+ this.afterEach = params.afterEach;
10
+ this.onError = params.onError;
22
11
  }
23
- const client = this.createDocumentClient({
24
- region: deployment.region
25
- });
26
- const retry = createRetry({
27
- maxRetries: this.maxRetries,
28
- retryDelay: this.retryDelay
29
- });
30
- for (const item of items) {
31
- const result = await retry.retry(async () => {
32
- switch (command) {
33
- case "delete":
34
- return await client.send(new DeleteCommand({
35
- TableName: table.name,
36
- Key: {
37
- PK: item.PK,
38
- SK: item.SK
39
- }
40
- }));
41
- case "put":
42
- return await client.send(new PutCommand({
43
- TableName: table.name,
44
- Item: item
45
- }));
46
- default:
47
- console.error(`Unsupported command type: ${command}`);
48
- return null;
49
- }
50
- }, {
51
- onFail: async error => {
52
- console.error("Error executing batch write command.");
53
- console.log(convertException(error));
54
- if (!this.onError) {
55
- return;
56
- }
57
- try {
58
- await this.onError({
59
- item,
60
- command,
61
- table,
62
- error
63
- });
64
- } catch (ex) {
65
- console.error(`Error in onError callback for command: ${command}`);
66
- console.log({
67
- original: convertException(error),
68
- error: convertException(ex)
12
+ async store(params) {
13
+ const { deployment, bundle, table, command, items } = params;
14
+ if (0 === items.length) return;
15
+ const client = this.createDocumentClient({
16
+ region: deployment.region
17
+ });
18
+ const retry = createRetry({
19
+ maxRetries: this.maxRetries,
20
+ retryDelay: this.retryDelay
21
+ });
22
+ for (const item of items){
23
+ const result = await retry.retry(async ()=>{
24
+ switch(command){
25
+ case "delete":
26
+ return await client.send(new DeleteCommand({
27
+ TableName: table.name,
28
+ Key: {
29
+ PK: item.PK,
30
+ SK: item.SK
31
+ }
32
+ }));
33
+ case "put":
34
+ return await client.send(new PutCommand({
35
+ TableName: table.name,
36
+ Item: item
37
+ }));
38
+ default:
39
+ console.error(`Unsupported command type: ${command}`);
40
+ return null;
41
+ }
42
+ }, {
43
+ onFail: async (error)=>{
44
+ console.error("Error executing batch write command.");
45
+ console.log(convertException(error));
46
+ if (!this.onError) return;
47
+ try {
48
+ await this.onError({
49
+ item,
50
+ command,
51
+ table,
52
+ error
53
+ });
54
+ } catch (ex) {
55
+ console.error(`Error in onError callback for command: ${command}`);
56
+ console.log({
57
+ original: convertException(error),
58
+ error: convertException(ex)
59
+ });
60
+ }
61
+ }
69
62
  });
70
- }
63
+ if (result && this.afterEach) try {
64
+ await this.afterEach({
65
+ table,
66
+ command,
67
+ item,
68
+ source: bundle.source,
69
+ target: deployment
70
+ });
71
+ } catch (ex) {
72
+ console.error(`Error in afterEach callback for command: ${command}`);
73
+ console.log(convertException(ex));
74
+ }
71
75
  }
72
- });
73
- if (!result || !this.afterEach) {
74
- continue;
75
- }
76
- try {
77
- await this.afterEach({
78
- table,
79
- command,
80
- item,
81
- source: bundle.source,
82
- target: deployment
83
- });
84
- } catch (ex) {
85
- console.error(`Error in afterEach callback for command: ${command}`);
86
- console.log(convertException(ex));
87
- }
88
76
  }
89
- }
90
77
  }
91
- export const createStorer = params => {
92
- return new Storer(params);
93
- };
78
+ const createStorer = (params)=>new Storer(params);
79
+ export { Storer, createStorer };
94
80
 
95
81
  //# sourceMappingURL=Storer.js.map