@webiny/api-sync-system 0.0.0-unstable.e53eceafb5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +113 -0
- package/constants.d.ts +1 -0
- package/constants.js +9 -0
- package/constants.js.map +1 -0
- package/index.d.ts +3 -0
- package/index.js +40 -0
- package/index.js.map +1 -0
- package/package.json +45 -0
- package/resolver/app/RecordHandler.d.ts +34 -0
- package/resolver/app/RecordHandler.js +126 -0
- package/resolver/app/RecordHandler.js.map +1 -0
- package/resolver/app/RecordsValidation.d.ts +6 -0
- package/resolver/app/RecordsValidation.js +29 -0
- package/resolver/app/RecordsValidation.js.map +1 -0
- package/resolver/app/ResolverApplication.d.ts +14 -0
- package/resolver/app/ResolverApplication.js +48 -0
- package/resolver/app/ResolverApplication.js.map +1 -0
- package/resolver/app/abstractions/RecordHandler.d.ts +7 -0
- package/resolver/app/abstractions/RecordHandler.js +7 -0
- package/resolver/app/abstractions/RecordHandler.js.map +1 -0
- package/resolver/app/abstractions/RecordsValidation.d.ts +14 -0
- package/resolver/app/abstractions/RecordsValidation.js +7 -0
- package/resolver/app/abstractions/RecordsValidation.js.map +1 -0
- package/resolver/app/abstractions/ResolverApplication.d.ts +7 -0
- package/resolver/app/abstractions/ResolverApplication.js +7 -0
- package/resolver/app/abstractions/ResolverApplication.js.map +1 -0
- package/resolver/app/abstractions/ResolverRecord.d.ts +42 -0
- package/resolver/app/abstractions/ResolverRecord.js +7 -0
- package/resolver/app/abstractions/ResolverRecord.js.map +1 -0
- package/resolver/app/bundler/BaseBundle.d.ts +21 -0
- package/resolver/app/bundler/BaseBundle.js +26 -0
- package/resolver/app/bundler/BaseBundle.js.map +1 -0
- package/resolver/app/bundler/Bundler.d.ts +10 -0
- package/resolver/app/bundler/Bundler.js +30 -0
- package/resolver/app/bundler/Bundler.js.map +1 -0
- package/resolver/app/bundler/Bundles.d.ts +14 -0
- package/resolver/app/bundler/Bundles.js +38 -0
- package/resolver/app/bundler/Bundles.js.map +1 -0
- package/resolver/app/bundler/CommandBundle.d.ts +15 -0
- package/resolver/app/bundler/CommandBundle.js +28 -0
- package/resolver/app/bundler/CommandBundle.js.map +1 -0
- package/resolver/app/bundler/TableBundle.d.ts +13 -0
- package/resolver/app/bundler/TableBundle.js +25 -0
- package/resolver/app/bundler/TableBundle.js.map +1 -0
- package/resolver/app/bundler/types.d.ts +29 -0
- package/resolver/app/bundler/types.js +7 -0
- package/resolver/app/bundler/types.js.map +1 -0
- package/resolver/app/commandHandler/DeleteCommandHandler.d.ts +18 -0
- package/resolver/app/commandHandler/DeleteCommandHandler.js +39 -0
- package/resolver/app/commandHandler/DeleteCommandHandler.js.map +1 -0
- package/resolver/app/commandHandler/PutCommandHandler.d.ts +18 -0
- package/resolver/app/commandHandler/PutCommandHandler.js +32 -0
- package/resolver/app/commandHandler/PutCommandHandler.js.map +1 -0
- package/resolver/app/commandHandler/delete.d.ts +2 -0
- package/resolver/app/commandHandler/delete.js +24 -0
- package/resolver/app/commandHandler/delete.js.map +1 -0
- package/resolver/app/commandHandler/put.d.ts +2 -0
- package/resolver/app/commandHandler/put.js +24 -0
- package/resolver/app/commandHandler/put.js.map +1 -0
- package/resolver/app/data/SourceDataContainer.d.ts +12 -0
- package/resolver/app/data/SourceDataContainer.js +42 -0
- package/resolver/app/data/SourceDataContainer.js.map +1 -0
- package/resolver/app/data/types.d.ts +23 -0
- package/resolver/app/data/types.js +7 -0
- package/resolver/app/data/types.js.map +1 -0
- package/resolver/app/fetcher/Fetcher.d.ts +22 -0
- package/resolver/app/fetcher/Fetcher.js +132 -0
- package/resolver/app/fetcher/Fetcher.js.map +1 -0
- package/resolver/app/fetcher/types.d.ts +25 -0
- package/resolver/app/fetcher/types.js +7 -0
- package/resolver/app/fetcher/types.js.map +1 -0
- package/resolver/app/ingestor/Ingestor.d.ts +13 -0
- package/resolver/app/ingestor/Ingestor.js +44 -0
- package/resolver/app/ingestor/Ingestor.js.map +1 -0
- package/resolver/app/ingestor/IngestorResult.d.ts +7 -0
- package/resolver/app/ingestor/IngestorResult.js +45 -0
- package/resolver/app/ingestor/IngestorResult.js.map +1 -0
- package/resolver/app/ingestor/types.d.ts +26 -0
- package/resolver/app/ingestor/types.js +7 -0
- package/resolver/app/ingestor/types.js.map +1 -0
- package/resolver/app/storer/Storer.d.ts +44 -0
- package/resolver/app/storer/Storer.js +103 -0
- package/resolver/app/storer/Storer.js.map +1 -0
- package/resolver/app/storer/types.d.ts +21 -0
- package/resolver/app/storer/types.js +7 -0
- package/resolver/app/storer/types.js.map +1 -0
- package/resolver/app/transform/TransformHandler.d.ts +34 -0
- package/resolver/app/transform/TransformHandler.js +70 -0
- package/resolver/app/transform/TransformHandler.js.map +1 -0
- package/resolver/app/transform/middleware.d.ts +3 -0
- package/resolver/app/transform/middleware.js +29 -0
- package/resolver/app/transform/middleware.js.map +1 -0
- package/resolver/app/utils/Retry.d.ts +18 -0
- package/resolver/app/utils/Retry.js +38 -0
- package/resolver/app/utils/Retry.js.map +1 -0
- package/resolver/app/utils/sleep.d.ts +1 -0
- package/resolver/app/utils/sleep.js +14 -0
- package/resolver/app/utils/sleep.js.map +1 -0
- package/resolver/app/validation/body.d.ts +142 -0
- package/resolver/app/validation/body.js +46 -0
- package/resolver/app/validation/body.js.map +1 -0
- package/resolver/app/validation/createJsonTransform.d.ts +2 -0
- package/resolver/app/validation/createJsonTransform.js +24 -0
- package/resolver/app/validation/createJsonTransform.js.map +1 -0
- package/resolver/app/validation/detail.d.ts +89 -0
- package/resolver/app/validation/detail.js +50 -0
- package/resolver/app/validation/detail.js.map +1 -0
- package/resolver/app/validation/event.d.ts +217 -0
- package/resolver/app/validation/event.js +32 -0
- package/resolver/app/validation/event.js.map +1 -0
- package/resolver/app/validation/numericString.d.ts +2 -0
- package/resolver/app/validation/numericString.js +23 -0
- package/resolver/app/validation/numericString.js.map +1 -0
- package/resolver/app/validation/system.d.ts +20 -0
- package/resolver/app/validation/system.js +20 -0
- package/resolver/app/validation/system.js.map +1 -0
- package/resolver/createEventHandlerPlugin.d.ts +10 -0
- package/resolver/createEventHandlerPlugin.js +127 -0
- package/resolver/createEventHandlerPlugin.js.map +1 -0
- package/resolver/createResolverHandler.d.ts +21 -0
- package/resolver/createResolverHandler.js +79 -0
- package/resolver/createResolverHandler.js.map +1 -0
- package/resolver/deployment/Deployment.d.ts +17 -0
- package/resolver/deployment/Deployment.js +50 -0
- package/resolver/deployment/Deployment.js.map +1 -0
- package/resolver/deployment/Deployments.d.ts +13 -0
- package/resolver/deployment/Deployments.js +34 -0
- package/resolver/deployment/Deployments.js.map +1 -0
- package/resolver/deployment/DeploymentsFetcher.d.ts +16 -0
- package/resolver/deployment/DeploymentsFetcher.js +112 -0
- package/resolver/deployment/DeploymentsFetcher.js.map +1 -0
- package/resolver/deployment/types.d.ts +34 -0
- package/resolver/deployment/types.js +7 -0
- package/resolver/deployment/types.js.map +1 -0
- package/resolver/lambda/LambdaTrigger.d.ts +18 -0
- package/resolver/lambda/LambdaTrigger.js +39 -0
- package/resolver/lambda/LambdaTrigger.js.map +1 -0
- package/resolver/plugins/CommandHandlerPlugin.d.ts +31 -0
- package/resolver/plugins/CommandHandlerPlugin.js +27 -0
- package/resolver/plugins/CommandHandlerPlugin.js.map +1 -0
- package/resolver/plugins/StorerAfterEachPlugin.d.ts +31 -0
- package/resolver/plugins/StorerAfterEachPlugin.js +33 -0
- package/resolver/plugins/StorerAfterEachPlugin.js.map +1 -0
- package/resolver/plugins/TransformRecordPlugin.d.ts +38 -0
- package/resolver/plugins/TransformRecordPlugin.js +30 -0
- package/resolver/plugins/TransformRecordPlugin.js.map +1 -0
- package/resolver/recordTypes/fileManager/CopyFile.d.ts +14 -0
- package/resolver/recordTypes/fileManager/CopyFile.js +81 -0
- package/resolver/recordTypes/fileManager/CopyFile.js.map +1 -0
- package/resolver/recordTypes/fileManager/DeleteFile.d.ts +14 -0
- package/resolver/recordTypes/fileManager/DeleteFile.js +72 -0
- package/resolver/recordTypes/fileManager/DeleteFile.js.map +1 -0
- package/resolver/recordTypes/fileManager/fileManager.d.ts +6 -0
- package/resolver/recordTypes/fileManager/fileManager.js +14 -0
- package/resolver/recordTypes/fileManager/fileManager.js.map +1 -0
- package/resolver/recordTypes/fileManager/fileManagerOnDelete.d.ts +5 -0
- package/resolver/recordTypes/fileManager/fileManagerOnDelete.js +48 -0
- package/resolver/recordTypes/fileManager/fileManagerOnDelete.js.map +1 -0
- package/resolver/recordTypes/fileManager/fileManagerOnPut.d.ts +5 -0
- package/resolver/recordTypes/fileManager/fileManagerOnPut.js +48 -0
- package/resolver/recordTypes/fileManager/fileManagerOnPut.js.map +1 -0
- package/resolver/recordTypes/fileManager/shouldBeHandled.d.ts +2 -0
- package/resolver/recordTypes/fileManager/shouldBeHandled.js +37 -0
- package/resolver/recordTypes/fileManager/shouldBeHandled.js.map +1 -0
- package/resolver/recordTypes/fileManager/types.d.ts +26 -0
- package/resolver/recordTypes/fileManager/types.js +7 -0
- package/resolver/recordTypes/fileManager/types.js.map +1 -0
- package/resolver/recordTypes/users/CopyUser.d.ts +18 -0
- package/resolver/recordTypes/users/CopyUser.js +67 -0
- package/resolver/recordTypes/users/CopyUser.js.map +1 -0
- package/resolver/recordTypes/users/DeleteUser.d.ts +18 -0
- package/resolver/recordTypes/users/DeleteUser.js +65 -0
- package/resolver/recordTypes/users/DeleteUser.js.map +1 -0
- package/resolver/recordTypes/users/shouldBeHandled.d.ts +2 -0
- package/resolver/recordTypes/users/shouldBeHandled.js +36 -0
- package/resolver/recordTypes/users/shouldBeHandled.js.map +1 -0
- package/resolver/recordTypes/users/types.d.ts +24 -0
- package/resolver/recordTypes/users/types.js +7 -0
- package/resolver/recordTypes/users/types.js.map +1 -0
- package/resolver/recordTypes/users/users.d.ts +6 -0
- package/resolver/recordTypes/users/users.js +14 -0
- package/resolver/recordTypes/users/users.js.map +1 -0
- package/resolver/recordTypes/users/usersOnDelete.d.ts +5 -0
- package/resolver/recordTypes/users/usersOnDelete.js +47 -0
- package/resolver/recordTypes/users/usersOnDelete.js.map +1 -0
- package/resolver/recordTypes/users/usersOnPut.d.ts +5 -0
- package/resolver/recordTypes/users/usersOnPut.js +48 -0
- package/resolver/recordTypes/users/usersOnPut.js.map +1 -0
- package/sync/FilterOutRecord.d.ts +11 -0
- package/sync/FilterOutRecord.js +29 -0
- package/sync/FilterOutRecord.js.map +1 -0
- package/sync/attachToDynamoDbDocument.d.ts +11 -0
- package/sync/attachToDynamoDbDocument.js +79 -0
- package/sync/attachToDynamoDbDocument.js.map +1 -0
- package/sync/createHandler.d.ts +14 -0
- package/sync/createHandler.js +63 -0
- package/sync/createHandler.js.map +1 -0
- package/sync/createSendDataToEventBridgeOnRequestEnd.d.ts +2 -0
- package/sync/createSendDataToEventBridgeOnRequestEnd.js +51 -0
- package/sync/createSendDataToEventBridgeOnRequestEnd.js.map +1 -0
- package/sync/createSyncSystem.d.ts +16 -0
- package/sync/createSyncSystem.js +42 -0
- package/sync/createSyncSystem.js.map +1 -0
- package/sync/filter/createDefaultFilterOutRecordPlugins.d.ts +13 -0
- package/sync/filter/createDefaultFilterOutRecordPlugins.js +88 -0
- package/sync/filter/createDefaultFilterOutRecordPlugins.js.map +1 -0
- package/sync/handler/Handler.d.ts +28 -0
- package/sync/handler/Handler.js +93 -0
- package/sync/handler/Handler.js.map +1 -0
- package/sync/handler/HandlerConverter.d.ts +12 -0
- package/sync/handler/HandlerConverter.js +43 -0
- package/sync/handler/HandlerConverter.js.map +1 -0
- package/sync/handler/converter/BatchGetCommandConverter.d.ts +8 -0
- package/sync/handler/converter/BatchGetCommandConverter.js +24 -0
- package/sync/handler/converter/BatchGetCommandConverter.js.map +1 -0
- package/sync/handler/converter/BatchWriteCommandConverter.d.ts +9 -0
- package/sync/handler/converter/BatchWriteCommandConverter.js +24 -0
- package/sync/handler/converter/BatchWriteCommandConverter.js.map +1 -0
- package/sync/handler/converter/DeleteCommandConverter.d.ts +9 -0
- package/sync/handler/converter/DeleteCommandConverter.js +24 -0
- package/sync/handler/converter/DeleteCommandConverter.js.map +1 -0
- package/sync/handler/converter/GetCommandConverter.d.ts +8 -0
- package/sync/handler/converter/GetCommandConverter.js +24 -0
- package/sync/handler/converter/GetCommandConverter.js.map +1 -0
- package/sync/handler/converter/PutCommandConverter.d.ts +9 -0
- package/sync/handler/converter/PutCommandConverter.js +24 -0
- package/sync/handler/converter/PutCommandConverter.js.map +1 -0
- package/sync/handler/converter/QueryCommandConverter.d.ts +8 -0
- package/sync/handler/converter/QueryCommandConverter.js +24 -0
- package/sync/handler/converter/QueryCommandConverter.js.map +1 -0
- package/sync/handler/converter/ScanCommandConverter.d.ts +8 -0
- package/sync/handler/converter/ScanCommandConverter.js +24 -0
- package/sync/handler/converter/ScanCommandConverter.js.map +1 -0
- package/sync/handler/converter/UpdateCommandConverter.d.ts +9 -0
- package/sync/handler/converter/UpdateCommandConverter.js +24 -0
- package/sync/handler/converter/UpdateCommandConverter.js.map +1 -0
- package/sync/handler/converter/commands/BatchWriteCommandValue.d.ts +9 -0
- package/sync/handler/converter/commands/BatchWriteCommandValue.js +51 -0
- package/sync/handler/converter/commands/BatchWriteCommandValue.js.map +1 -0
- package/sync/handler/converter/commands/DeleteCommandValue.d.ts +9 -0
- package/sync/handler/converter/commands/DeleteCommandValue.js +27 -0
- package/sync/handler/converter/commands/DeleteCommandValue.js.map +1 -0
- package/sync/handler/converter/commands/NullCommandValue.d.ts +8 -0
- package/sync/handler/converter/commands/NullCommandValue.js +18 -0
- package/sync/handler/converter/commands/NullCommandValue.js.map +1 -0
- package/sync/handler/converter/commands/PutCommandValue.d.ts +9 -0
- package/sync/handler/converter/commands/PutCommandValue.js +27 -0
- package/sync/handler/converter/commands/PutCommandValue.js.map +1 -0
- package/sync/handler/converter/commands/UpdateCommandValue.d.ts +9 -0
- package/sync/handler/converter/commands/UpdateCommandValue.js +27 -0
- package/sync/handler/converter/commands/UpdateCommandValue.js.map +1 -0
- package/sync/handler/types.d.ts +18 -0
- package/sync/handler/types.js +7 -0
- package/sync/handler/types.js.map +1 -0
- package/sync/plugins/FilterOutRecordPlugin.d.ts +23 -0
- package/sync/plugins/FilterOutRecordPlugin.js +31 -0
- package/sync/plugins/FilterOutRecordPlugin.js.map +1 -0
- package/sync/requestPlugin.d.ts +9 -0
- package/sync/requestPlugin.js +59 -0
- package/sync/requestPlugin.js.map +1 -0
- package/sync/types.d.ts +64 -0
- package/sync/types.js +7 -0
- package/sync/types.js.map +1 -0
- package/sync/utils/getTableType.d.ts +2 -0
- package/sync/utils/getTableType.js +22 -0
- package/sync/utils/getTableType.js.map +1 -0
- package/sync/utils/manifest.d.ts +17 -0
- package/sync/utils/manifest.js +52 -0
- package/sync/utils/manifest.js.map +1 -0
- package/sync/utils/validateSystemInput.d.ts +11 -0
- package/sync/utils/validateSystemInput.js +44 -0
- package/sync/utils/validateSystemInput.js.map +1 -0
- package/types.d.ts +42 -0
- package/types.js +15 -0
- package/types.js.map +1 -0
- package/utils/createSystemName.d.ts +5 -0
- package/utils/createSystemName.js +12 -0
- package/utils/createSystemName.js.map +1 -0
- package/worker/actions/copyFile/CopyFile.d.ts +63 -0
- package/worker/actions/copyFile/CopyFile.js +283 -0
- package/worker/actions/copyFile/CopyFile.js.map +1 -0
- package/worker/actions/copyFile/copyFileAction.d.ts +6 -0
- package/worker/actions/copyFile/copyFileAction.js +47 -0
- package/worker/actions/copyFile/copyFileAction.js.map +1 -0
- package/worker/actions/copyFile/copyFileSchema.d.ts +47 -0
- package/worker/actions/copyFile/copyFileSchema.js +25 -0
- package/worker/actions/copyFile/copyFileSchema.js.map +1 -0
- package/worker/actions/copyFile/types.d.ts +10 -0
- package/worker/actions/copyFile/types.js +7 -0
- package/worker/actions/copyFile/types.js.map +1 -0
- package/worker/actions/createUser/CreateUser.d.ts +17 -0
- package/worker/actions/createUser/CreateUser.js +90 -0
- package/worker/actions/createUser/CreateUser.js.map +1 -0
- package/worker/actions/createUser/createUserAction.d.ts +6 -0
- package/worker/actions/createUser/createUserAction.js +42 -0
- package/worker/actions/createUser/createUserAction.js.map +1 -0
- package/worker/actions/createUser/createUserSchema.d.ts +47 -0
- package/worker/actions/createUser/createUserSchema.js +25 -0
- package/worker/actions/createUser/createUserSchema.js.map +1 -0
- package/worker/actions/createUser/types.d.ts +10 -0
- package/worker/actions/createUser/types.js +7 -0
- package/worker/actions/createUser/types.js.map +1 -0
- package/worker/actions/deleteFile/DeleteFile.d.ts +24 -0
- package/worker/actions/deleteFile/DeleteFile.js +82 -0
- package/worker/actions/deleteFile/DeleteFile.js.map +1 -0
- package/worker/actions/deleteFile/deleteFileAction.d.ts +6 -0
- package/worker/actions/deleteFile/deleteFileAction.js +42 -0
- package/worker/actions/deleteFile/deleteFileAction.js.map +1 -0
- package/worker/actions/deleteFile/deleteFileSchema.d.ts +47 -0
- package/worker/actions/deleteFile/deleteFileSchema.js +25 -0
- package/worker/actions/deleteFile/deleteFileSchema.js.map +1 -0
- package/worker/actions/deleteFile/types.d.ts +9 -0
- package/worker/actions/deleteFile/types.js +7 -0
- package/worker/actions/deleteFile/types.js.map +1 -0
- package/worker/actions/deleteUser/DeleteUser.d.ts +16 -0
- package/worker/actions/deleteUser/DeleteUser.js +67 -0
- package/worker/actions/deleteUser/DeleteUser.js.map +1 -0
- package/worker/actions/deleteUser/deleteUserAction.d.ts +6 -0
- package/worker/actions/deleteUser/deleteUserAction.js +40 -0
- package/worker/actions/deleteUser/deleteUserAction.js.map +1 -0
- package/worker/actions/deleteUser/deleteUserSchema.d.ts +29 -0
- package/worker/actions/deleteUser/deleteUserSchema.js +24 -0
- package/worker/actions/deleteUser/deleteUserSchema.js.map +1 -0
- package/worker/actions/deleteUser/types.d.ts +9 -0
- package/worker/actions/deleteUser/types.js +7 -0
- package/worker/actions/deleteUser/types.js.map +1 -0
- package/worker/actions/logValidationError.d.ts +2 -0
- package/worker/actions/logValidationError.js +17 -0
- package/worker/actions/logValidationError.js.map +1 -0
- package/worker/actions/removeCognitoUserAttributes.d.ts +6 -0
- package/worker/actions/removeCognitoUserAttributes.js +24 -0
- package/worker/actions/removeCognitoUserAttributes.js.map +1 -0
- package/worker/actions/updateUser/UpdateUser.d.ts +17 -0
- package/worker/actions/updateUser/UpdateUser.js +81 -0
- package/worker/actions/updateUser/UpdateUser.js.map +1 -0
- package/worker/actions/updateUser/types.d.ts +10 -0
- package/worker/actions/updateUser/types.js +7 -0
- package/worker/actions/updateUser/types.js.map +1 -0
- package/worker/actions/updateUser/updateUserAction.d.ts +6 -0
- package/worker/actions/updateUser/updateUserAction.js +42 -0
- package/worker/actions/updateUser/updateUserAction.js.map +1 -0
- package/worker/actions/updateUser/updateUserSchema.d.ts +47 -0
- package/worker/actions/updateUser/updateUserSchema.js +25 -0
- package/worker/actions/updateUser/updateUserSchema.js.map +1 -0
- package/worker/createWorkerHandler.d.ts +11 -0
- package/worker/createWorkerHandler.js +48 -0
- package/worker/createWorkerHandler.js.map +1 -0
- package/worker/handler/WorkerActionHandler.d.ts +10 -0
- package/worker/handler/WorkerActionHandler.js +33 -0
- package/worker/handler/WorkerActionHandler.js.map +1 -0
- package/worker/handler/eventHandler.d.ts +1 -0
- package/worker/handler/eventHandler.js +31 -0
- package/worker/handler/eventHandler.js.map +1 -0
- package/worker/plugins/WorkerActionPlugin.d.ts +10 -0
- package/worker/plugins/WorkerActionPlugin.js +28 -0
- package/worker/plugins/WorkerActionPlugin.js.map +1 -0
- package/worker/types.d.ts +14 -0
- package/worker/types.js +7 -0
- package/worker/types.js.map +1 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { ITable } from "../../../sync/types.js";
|
|
2
|
+
import type { IDeployment } from "../../deployment/types.js";
|
|
3
|
+
import type { GenericRecord } from "@webiny/api/types.js";
|
|
4
|
+
import type { IStoreItem } from "../storer/types.js";
|
|
5
|
+
export interface IInputItem {
|
|
6
|
+
PK: string;
|
|
7
|
+
SK: string;
|
|
8
|
+
table: Pick<ITable, "name">;
|
|
9
|
+
source: Pick<IDeployment, "name">;
|
|
10
|
+
}
|
|
11
|
+
export interface IItem {
|
|
12
|
+
PK: string;
|
|
13
|
+
SK: string;
|
|
14
|
+
table: Pick<ITable, "name">;
|
|
15
|
+
source: Pick<IDeployment, "name">;
|
|
16
|
+
data: IStoreItem | null;
|
|
17
|
+
}
|
|
18
|
+
export interface ISourceDataContainer {
|
|
19
|
+
items: GenericRecord<string, IItem>;
|
|
20
|
+
get(item: IInputItem): IStoreItem | null;
|
|
21
|
+
add(item: IInputItem, data: IStoreItem | null): void;
|
|
22
|
+
merge(container: ISourceDataContainer): void;
|
|
23
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"names":[],"sources":["types.ts"],"sourcesContent":["import type { ITable } from \"~/sync/types.js\";\nimport type { IDeployment } from \"~/resolver/deployment/types.js\";\nimport type { GenericRecord } from \"@webiny/api/types.js\";\nimport type { IStoreItem } from \"~/resolver/app/storer/types.js\";\n\nexport interface IInputItem {\n PK: string;\n SK: string;\n table: Pick<ITable, \"name\">;\n source: Pick<IDeployment, \"name\">;\n}\n\nexport interface IItem {\n PK: string;\n SK: string;\n table: Pick<ITable, \"name\">;\n source: Pick<IDeployment, \"name\">;\n data: IStoreItem | null;\n}\n\nexport interface ISourceDataContainer {\n items: GenericRecord<string, IItem>;\n get(item: IInputItem): IStoreItem | null;\n add(item: IInputItem, data: IStoreItem | null): void;\n merge(container: ISourceDataContainer): void;\n}\n"],"mappings":"","ignoreList":[]}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { IFetcher, IFetcherExecParams, IFetcherExecResult } from "./types";
|
|
2
|
+
import { type DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb";
|
|
3
|
+
import type { IDeployment } from "../../deployment/types.js";
|
|
4
|
+
export interface IFetcherParamsCreateDocumentClientCallable {
|
|
5
|
+
(deployment: Pick<IDeployment, "region">): Pick<DynamoDBDocument, "send">;
|
|
6
|
+
}
|
|
7
|
+
export interface IFetcherParams {
|
|
8
|
+
maxRetries?: number;
|
|
9
|
+
retryDelay?: number;
|
|
10
|
+
createDocumentClient: IFetcherParamsCreateDocumentClientCallable;
|
|
11
|
+
}
|
|
12
|
+
export declare class Fetcher implements IFetcher {
|
|
13
|
+
private readonly createDocumentClient;
|
|
14
|
+
private readonly maxRetries;
|
|
15
|
+
private readonly retryDelay;
|
|
16
|
+
constructor(params: IFetcherParams);
|
|
17
|
+
exec(params: IFetcherExecParams): Promise<IFetcherExecResult>;
|
|
18
|
+
private execute;
|
|
19
|
+
private runCommand;
|
|
20
|
+
private getKeys;
|
|
21
|
+
}
|
|
22
|
+
export declare const createFetcher: (params: IFetcherParams) => IFetcher;
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault").default;
|
|
4
|
+
Object.defineProperty(exports, "__esModule", {
|
|
5
|
+
value: true
|
|
6
|
+
});
|
|
7
|
+
exports.createFetcher = exports.Fetcher = void 0;
|
|
8
|
+
var _clientDynamodb = require("@webiny/aws-sdk/client-dynamodb");
|
|
9
|
+
var _chunk = _interopRequireDefault(require("lodash/chunk"));
|
|
10
|
+
var _SourceDataContainer = require("../data/SourceDataContainer.js");
|
|
11
|
+
var _utils = require("@webiny/utils");
|
|
12
|
+
var _Retry = require("../utils/Retry.js");
|
|
13
|
+
class Fetcher {
|
|
14
|
+
constructor(params) {
|
|
15
|
+
this.createDocumentClient = params.createDocumentClient;
|
|
16
|
+
this.maxRetries = params.maxRetries || 10;
|
|
17
|
+
this.retryDelay = params.retryDelay || 1000;
|
|
18
|
+
}
|
|
19
|
+
async exec(params) {
|
|
20
|
+
const {
|
|
21
|
+
deployment,
|
|
22
|
+
items: input,
|
|
23
|
+
table,
|
|
24
|
+
maxBatchSize = 25
|
|
25
|
+
} = params;
|
|
26
|
+
if (input.length === 0) {
|
|
27
|
+
return {
|
|
28
|
+
items: _SourceDataContainer.SourceDataContainer.create()
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
const client = this.createDocumentClient(deployment);
|
|
32
|
+
const results = await this.execute({
|
|
33
|
+
client,
|
|
34
|
+
table,
|
|
35
|
+
maxBatchSize,
|
|
36
|
+
records: input
|
|
37
|
+
});
|
|
38
|
+
const items = _SourceDataContainer.SourceDataContainer.create();
|
|
39
|
+
for (const item of input) {
|
|
40
|
+
const data = results.find(result => {
|
|
41
|
+
return item.PK === result.PK && item.SK === result.SK;
|
|
42
|
+
});
|
|
43
|
+
items.add({
|
|
44
|
+
PK: item.PK,
|
|
45
|
+
SK: item.SK,
|
|
46
|
+
table,
|
|
47
|
+
source: deployment
|
|
48
|
+
}, data || null);
|
|
49
|
+
}
|
|
50
|
+
return {
|
|
51
|
+
items
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
async execute(params) {
|
|
55
|
+
const {
|
|
56
|
+
client,
|
|
57
|
+
table,
|
|
58
|
+
records: items,
|
|
59
|
+
maxBatchSize
|
|
60
|
+
} = params;
|
|
61
|
+
const batches = (0, _chunk.default)(items, maxBatchSize);
|
|
62
|
+
const results = [];
|
|
63
|
+
for (const batch of batches) {
|
|
64
|
+
let keys = this.getKeys(batch);
|
|
65
|
+
while (keys.length > 0) {
|
|
66
|
+
const command = new _clientDynamodb.BatchGetCommand({
|
|
67
|
+
RequestItems: {
|
|
68
|
+
[table.name]: {
|
|
69
|
+
Keys: keys
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
});
|
|
73
|
+
const {
|
|
74
|
+
items,
|
|
75
|
+
unprocessedKeys
|
|
76
|
+
} = await this.runCommand({
|
|
77
|
+
command,
|
|
78
|
+
client,
|
|
79
|
+
table: table.name
|
|
80
|
+
});
|
|
81
|
+
results.push(...items);
|
|
82
|
+
keys = unprocessedKeys;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
return results;
|
|
86
|
+
}
|
|
87
|
+
async runCommand(params) {
|
|
88
|
+
const {
|
|
89
|
+
command,
|
|
90
|
+
table,
|
|
91
|
+
client
|
|
92
|
+
} = params;
|
|
93
|
+
const retry = (0, _Retry.createRetry)({
|
|
94
|
+
maxRetries: this.maxRetries,
|
|
95
|
+
retryDelay: this.retryDelay
|
|
96
|
+
});
|
|
97
|
+
return await retry.retry(async () => {
|
|
98
|
+
const result = await client.send(command);
|
|
99
|
+
return {
|
|
100
|
+
items: result.Responses?.[table] || [],
|
|
101
|
+
unprocessedKeys: result.UnprocessedKeys?.[table]?.Keys || []
|
|
102
|
+
};
|
|
103
|
+
}, {
|
|
104
|
+
onFail: ex => {
|
|
105
|
+
console.error(`Max retries reached. Could not fetch items from table: ${table}`);
|
|
106
|
+
console.log((0, _utils.convertException)(ex));
|
|
107
|
+
console.log(JSON.stringify(command));
|
|
108
|
+
}
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
getKeys(items) {
|
|
112
|
+
const keys = new Map();
|
|
113
|
+
for (const item of items) {
|
|
114
|
+
const key = `PK:${item.PK}#SK:${item.SK}`;
|
|
115
|
+
if (keys.has(key)) {
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
keys.set(key, {
|
|
119
|
+
PK: item.PK,
|
|
120
|
+
SK: item.SK
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
return Array.from(keys.values());
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
exports.Fetcher = Fetcher;
|
|
127
|
+
const createFetcher = params => {
|
|
128
|
+
return new Fetcher(params);
|
|
129
|
+
};
|
|
130
|
+
exports.createFetcher = createFetcher;
|
|
131
|
+
|
|
132
|
+
//# sourceMappingURL=Fetcher.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"names":["_clientDynamodb","require","_chunk","_interopRequireDefault","_SourceDataContainer","_utils","_Retry","Fetcher","constructor","params","createDocumentClient","maxRetries","retryDelay","exec","deployment","items","input","table","maxBatchSize","length","SourceDataContainer","create","client","results","execute","records","item","data","find","result","PK","SK","add","source","batches","lodashChunk","batch","keys","getKeys","command","BatchGetCommand","RequestItems","name","Keys","unprocessedKeys","runCommand","push","retry","createRetry","send","Responses","UnprocessedKeys","onFail","ex","console","error","log","convertException","JSON","stringify","Map","key","has","set","Array","from","values","exports","createFetcher"],"sources":["Fetcher.ts"],"sourcesContent":["import type { IFetcher, IFetcherExecParams, IFetcherExecResult } from \"./types\";\nimport { BatchGetCommand, type DynamoDBDocument } from \"@webiny/aws-sdk/client-dynamodb\";\nimport lodashChunk from \"lodash/chunk\";\nimport type { IDeployment } from \"~/resolver/deployment/types.js\";\nimport { SourceDataContainer } from \"~/resolver/app/data/SourceDataContainer.js\";\nimport type { GenericRecord } from \"@webiny/api/types.js\";\nimport { convertException } from \"@webiny/utils\";\nimport type { IStoreItem } from \"~/resolver/app/storer/types.js\";\nimport type { ITable } from \"~/sync/types.js\";\nimport { createRetry } from \"~/resolver/app/utils/Retry.js\";\n\nexport interface IFetcherParamsCreateDocumentClientCallable {\n (deployment: Pick<IDeployment, \"region\">): Pick<DynamoDBDocument, \"send\">;\n}\n\nexport interface IFetcherParams {\n maxRetries?: number;\n retryDelay?: number;\n createDocumentClient: IFetcherParamsCreateDocumentClientCallable;\n}\n\ninterface IKeys {\n PK: string;\n SK: string;\n}\n\ninterface IFetcherExecuteRunCommandParams {\n command: BatchGetCommand;\n table: string;\n client: Pick<DynamoDBDocument, \"send\">;\n}\n\ninterface IFetcherExecuteRunCommandResult<T = GenericRecord> {\n items: T[];\n unprocessedKeys: IKeys[];\n}\n\ninterface IFetchExecuteExecuteParamsItem {\n PK: string;\n SK: string;\n}\n\ninterface IFetchExecuteExecuteParams {\n maxBatchSize: number;\n client: Pick<DynamoDBDocument, \"send\">;\n table: ITable;\n records: IFetchExecuteExecuteParamsItem[];\n}\n\nexport class Fetcher implements IFetcher {\n private readonly createDocumentClient: IFetcherParamsCreateDocumentClientCallable;\n private readonly maxRetries: number;\n private readonly retryDelay: number;\n\n public constructor(params: IFetcherParams) {\n this.createDocumentClient = params.createDocumentClient;\n this.maxRetries = params.maxRetries || 10;\n this.retryDelay = params.retryDelay || 1000;\n }\n\n public async exec(params: IFetcherExecParams): Promise<IFetcherExecResult> {\n const { deployment, items: input, table, maxBatchSize = 25 } = params;\n if (input.length === 0) {\n return {\n items: SourceDataContainer.create()\n };\n }\n\n const client = this.createDocumentClient(deployment);\n\n const results = await this.execute({\n client,\n table,\n maxBatchSize,\n records: input\n });\n\n const items = SourceDataContainer.create();\n\n for (const item of input) {\n const data = results.find(result => {\n return item.PK === result.PK && item.SK === result.SK;\n });\n items.add(\n {\n PK: item.PK,\n SK: item.SK,\n table,\n source: deployment\n },\n data || null\n );\n }\n\n return {\n items\n };\n }\n\n private async execute<T = IStoreItem>(params: IFetchExecuteExecuteParams) {\n const { client, table, records: items, maxBatchSize } = params;\n const batches = lodashChunk(items, maxBatchSize);\n\n const results: T[] = [];\n for (const batch of batches) {\n let keys = this.getKeys(batch);\n\n while (keys.length > 0) {\n const command = new BatchGetCommand({\n RequestItems: {\n [table.name]: {\n Keys: keys\n }\n }\n });\n const { items, unprocessedKeys } = await this.runCommand<T>({\n command,\n client,\n table: table.name\n });\n\n results.push(...items);\n\n keys = unprocessedKeys;\n }\n }\n return results;\n }\n\n private async runCommand<T = GenericRecord>(\n params: IFetcherExecuteRunCommandParams\n ): Promise<IFetcherExecuteRunCommandResult<T>> {\n const { command, table, client } = params;\n\n const retry = createRetry({\n maxRetries: this.maxRetries,\n retryDelay: this.retryDelay\n });\n\n return await retry.retry(\n async () => {\n const result = await client.send(command);\n\n return {\n items: (result.Responses?.[table] || []) as T[],\n unprocessedKeys: (result.UnprocessedKeys?.[table]?.Keys || []) as IKeys[]\n };\n },\n {\n onFail: ex => {\n console.error(\n `Max retries reached. Could not fetch items from table: ${table}`\n );\n console.log(convertException(ex));\n console.log(JSON.stringify(command));\n }\n }\n );\n }\n\n private getKeys(items: IFetchExecuteExecuteParamsItem[]): IKeys[] {\n const keys = new Map<string, IKeys>();\n\n for (const item of items) {\n const key = `PK:${item.PK}#SK:${item.SK}`;\n if (keys.has(key)) {\n continue;\n }\n keys.set(key, {\n PK: item.PK,\n SK: item.SK\n });\n }\n\n return Array.from(keys.values());\n }\n}\n\nexport const createFetcher = (params: IFetcherParams): IFetcher => {\n return new Fetcher(params);\n};\n"],"mappings":";;;;;;;AACA,IAAAA,eAAA,GAAAC,OAAA;AACA,IAAAC,MAAA,GAAAC,sBAAA,CAAAF,OAAA;AAEA,IAAAG,oBAAA,GAAAH,OAAA;AAEA,IAAAI,MAAA,GAAAJ,OAAA;AAGA,IAAAK,MAAA,GAAAL,OAAA;AAwCO,MAAMM,OAAO,CAAqB;EAK9BC,WAAWA,CAACC,MAAsB,EAAE;IACvC,IAAI,CAACC,oBAAoB,GAAGD,MAAM,CAACC,oBAAoB;IACvD,IAAI,CAACC,UAAU,GAAGF,MAAM,CAACE,UAAU,IAAI,EAAE;IACzC,IAAI,CAACC,UAAU,GAAGH,MAAM,CAACG,UAAU,IAAI,IAAI;EAC/C;EAEA,MAAaC,IAAIA,CAACJ,MAA0B,EAA+B;IACvE,MAAM;MAAEK,UAAU;MAAEC,KAAK,EAAEC,KAAK;MAAEC,KAAK;MAAEC,YAAY,GAAG;IAAG,CAAC,GAAGT,MAAM;IACrE,IAAIO,KAAK,CAACG,MAAM,KAAK,CAAC,EAAE;MACpB,OAAO;QACHJ,KAAK,EAAEK,wCAAmB,CAACC,MAAM,CAAC;MACtC,CAAC;IACL;IAEA,MAAMC,MAAM,GAAG,IAAI,CAACZ,oBAAoB,CAACI,UAAU,CAAC;IAEpD,MAAMS,OAAO,GAAG,MAAM,IAAI,CAACC,OAAO,CAAC;MAC/BF,MAAM;MACNL,KAAK;MACLC,YAAY;MACZO,OAAO,EAAET;IACb,CAAC,CAAC;IAEF,MAAMD,KAAK,GAAGK,wCAAmB,CAACC,MAAM,CAAC,CAAC;IAE1C,KAAK,MAAMK,IAAI,IAAIV,KAAK,EAAE;MACtB,MAAMW,IAAI,GAAGJ,OAAO,CAACK,IAAI,CAACC,MAAM,IAAI;QAChC,OAAOH,IAAI,CAACI,EAAE,KAAKD,MAAM,CAACC,EAAE,IAAIJ,IAAI,CAACK,EAAE,KAAKF,MAAM,CAACE,EAAE;MACzD,CAAC,CAAC;MACFhB,KAAK,CAACiB,GAAG,CACL;QACIF,EAAE,EAAEJ,IAAI,CAACI,EAAE;QACXC,EAAE,EAAEL,IAAI,CAACK,EAAE;QACXd,KAAK;QACLgB,MAAM,EAAEnB;MACZ,CAAC,EACDa,IAAI,IAAI,IACZ,CAAC;IACL;IAEA,OAAO;MACHZ;IACJ,CAAC;EACL;EAEA,MAAcS,OAAOA,CAAiBf,MAAkC,EAAE;IACtE,MAAM;MAAEa,MAAM;MAAEL,KAAK;MAAEQ,OAAO,EAAEV,KAAK;MAAEG;IAAa,CAAC,GAAGT,MAAM;IAC9D,MAAMyB,OAAO,GAAG,IAAAC,cAAW,EAACpB,KAAK,EAAEG,YAAY,CAAC;IAEhD,MAAMK,OAAY,GAAG,EAAE;IACvB,KAAK,MAAMa,KAAK,IAAIF,OAAO,EAAE;MACzB,IAAIG,IAAI,GAAG,IAAI,CAACC,OAAO,CAACF,KAAK,CAAC;MAE9B,OAAOC,IAAI,CAAClB,MAAM,GAAG,CAAC,EAAE;QACpB,MAAMoB,OAAO,GAAG,IAAIC,+BAAe,CAAC;UAChCC,YAAY,EAAE;YACV,CAACxB,KAAK,CAACyB,IAAI,GAAG;cACVC,IAAI,EAAEN;YACV;UACJ;QACJ,CAAC,CAAC;QACF,MAAM;UAAEtB,KAAK;UAAE6B;QAAgB,CAAC,GAAG,MAAM,IAAI,CAACC,UAAU,CAAI;UACxDN,OAAO;UACPjB,MAAM;UACNL,KAAK,EAAEA,KAAK,CAACyB;QACjB,CAAC,CAAC;QAEFnB,OAAO,CAACuB,IAAI,CAAC,GAAG/B,KAAK,CAAC;QAEtBsB,IAAI,GAAGO,eAAe;MAC1B;IACJ;IACA,OAAOrB,OAAO;EAClB;EAEA,MAAcsB,UAAUA,CACpBpC,MAAuC,EACI;IAC3C,MAAM;MAAE8B,OAAO;MAAEtB,KAAK;MAAEK;IAAO,CAAC,GAAGb,MAAM;IAEzC,MAAMsC,KAAK,GAAG,IAAAC,kBAAW,EAAC;MACtBrC,UAAU,EAAE,IAAI,CAACA,UAAU;MAC3BC,UAAU,EAAE,IAAI,CAACA;IACrB,CAAC,CAAC;IAEF,OAAO,MAAMmC,KAAK,CAACA,KAAK,CACpB,YAAY;MACR,MAAMlB,MAAM,GAAG,MAAMP,MAAM,CAAC2B,IAAI,CAACV,OAAO,CAAC;MAEzC,OAAO;QACHxB,KAAK,EAAGc,MAAM,CAACqB,SAAS,GAAGjC,KAAK,CAAC,IAAI,EAAU;QAC/C2B,eAAe,EAAGf,MAAM,CAACsB,eAAe,GAAGlC,KAAK,CAAC,EAAE0B,IAAI,IAAI;MAC/D,CAAC;IACL,CAAC,EACD;MACIS,MAAM,EAAEC,EAAE,IAAI;QACVC,OAAO,CAACC,KAAK,CACT,0DAA0DtC,KAAK,EACnE,CAAC;QACDqC,OAAO,CAACE,GAAG,CAAC,IAAAC,uBAAgB,EAACJ,EAAE,CAAC,CAAC;QACjCC,OAAO,CAACE,GAAG,CAACE,IAAI,CAACC,SAAS,CAACpB,OAAO,CAAC,CAAC;MACxC;IACJ,CACJ,CAAC;EACL;EAEQD,OAAOA,CAACvB,KAAuC,EAAW;IAC9D,MAAMsB,IAAI,GAAG,IAAIuB,GAAG,CAAgB,CAAC;IAErC,KAAK,MAAMlC,IAAI,IAAIX,KAAK,EAAE;MACtB,MAAM8C,GAAG,GAAG,MAAMnC,IAAI,CAACI,EAAE,OAAOJ,IAAI,CAACK,EAAE,EAAE;MACzC,IAAIM,IAAI,CAACyB,GAAG,CAACD,GAAG,CAAC,EAAE;QACf;MACJ;MACAxB,IAAI,CAAC0B,GAAG,CAACF,GAAG,EAAE;QACV/B,EAAE,EAAEJ,IAAI,CAACI,EAAE;QACXC,EAAE,EAAEL,IAAI,CAACK;MACb,CAAC,CAAC;IACN;IAEA,OAAOiC,KAAK,CAACC,IAAI,CAAC5B,IAAI,CAAC6B,MAAM,CAAC,CAAC,CAAC;EACpC;AACJ;AAACC,OAAA,CAAA5D,OAAA,GAAAA,OAAA;AAEM,MAAM6D,aAAa,GAAI3D,MAAsB,IAAe;EAC/D,OAAO,IAAIF,OAAO,CAACE,MAAM,CAAC;AAC9B,CAAC;AAAC0D,OAAA,CAAAC,aAAA,GAAAA,aAAA","ignoreList":[]}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { IDeployment } from "../../deployment/types.js";
|
|
2
|
+
import type { ITable } from "../../../sync/types.js";
|
|
3
|
+
import type { ISourceDataContainer } from "../data/types.js";
|
|
4
|
+
export interface IFetcherExecParamsItem {
|
|
5
|
+
PK: string;
|
|
6
|
+
SK: string;
|
|
7
|
+
}
|
|
8
|
+
export interface IFetcherExecParams {
|
|
9
|
+
maxBatchSize?: number;
|
|
10
|
+
deployment: IDeployment;
|
|
11
|
+
table: ITable;
|
|
12
|
+
items: IFetcherExecParamsItem[];
|
|
13
|
+
}
|
|
14
|
+
export interface IFetcherExecValidResult {
|
|
15
|
+
error?: never;
|
|
16
|
+
items: ISourceDataContainer;
|
|
17
|
+
}
|
|
18
|
+
export interface IFetcherExecErrorResult {
|
|
19
|
+
error: Error;
|
|
20
|
+
items?: never;
|
|
21
|
+
}
|
|
22
|
+
export type IFetcherExecResult = IFetcherExecValidResult | IFetcherExecErrorResult;
|
|
23
|
+
export interface IFetcher {
|
|
24
|
+
exec(params: IFetcherExecParams): Promise<IFetcherExecResult>;
|
|
25
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"names":[],"sources":["types.ts"],"sourcesContent":["import type { IDeployment } from \"~/resolver/deployment/types.js\";\nimport type { ITable } from \"~/sync/types.js\";\nimport type { ISourceDataContainer } from \"~/resolver/app/data/types.js\";\n\nexport interface IFetcherExecParamsItem {\n PK: string;\n SK: string;\n}\n\nexport interface IFetcherExecParams {\n maxBatchSize?: number;\n deployment: IDeployment;\n table: ITable;\n items: IFetcherExecParamsItem[];\n}\n\nexport interface IFetcherExecValidResult {\n error?: never;\n items: ISourceDataContainer;\n}\n\nexport interface IFetcherExecErrorResult {\n error: Error;\n items?: never;\n}\n\nexport type IFetcherExecResult = IFetcherExecValidResult | IFetcherExecErrorResult;\n\nexport interface IFetcher {\n exec(params: IFetcherExecParams): Promise<IFetcherExecResult>;\n}\n"],"mappings":"","ignoreList":[]}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { IIngestor, IIngestorIngestParams, IIngestorResult } from "./types.js";
|
|
2
|
+
import type { IDeployment } from "../../deployment/types.js";
|
|
3
|
+
export interface IIngestorParams {
|
|
4
|
+
createIngestorResult: () => IIngestorResult;
|
|
5
|
+
getSource: (name: string) => IDeployment | null;
|
|
6
|
+
}
|
|
7
|
+
export declare class Ingestor implements IIngestor {
|
|
8
|
+
private readonly createIngestorResult;
|
|
9
|
+
private readonly getSource;
|
|
10
|
+
constructor(params: IIngestorParams);
|
|
11
|
+
ingest(params: IIngestorIngestParams): Promise<IIngestorResult>;
|
|
12
|
+
}
|
|
13
|
+
export declare const createIngestor: (params: IIngestorParams) => IIngestor;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
Object.defineProperty(exports, "__esModule", {
|
|
4
|
+
value: true
|
|
5
|
+
});
|
|
6
|
+
exports.createIngestor = exports.Ingestor = void 0;
|
|
7
|
+
class Ingestor {
|
|
8
|
+
constructor(params) {
|
|
9
|
+
this.createIngestorResult = params.createIngestorResult;
|
|
10
|
+
this.getSource = params.getSource;
|
|
11
|
+
}
|
|
12
|
+
async ingest(params) {
|
|
13
|
+
const {
|
|
14
|
+
records
|
|
15
|
+
} = params;
|
|
16
|
+
const result = this.createIngestorResult();
|
|
17
|
+
for (const record of records) {
|
|
18
|
+
const detail = record.body.detail;
|
|
19
|
+
const source = this.getSource(detail.source.name);
|
|
20
|
+
if (!source) {
|
|
21
|
+
console.error(`Could not find deployment for SQS Record source: ${detail.source.name}. More info in next log line.`);
|
|
22
|
+
console.log(JSON.stringify({
|
|
23
|
+
source: detail.source,
|
|
24
|
+
items: detail.items
|
|
25
|
+
}));
|
|
26
|
+
continue;
|
|
27
|
+
}
|
|
28
|
+
for (const item of detail.items) {
|
|
29
|
+
result.add({
|
|
30
|
+
item,
|
|
31
|
+
source
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
return result;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
exports.Ingestor = Ingestor;
|
|
39
|
+
const createIngestor = params => {
|
|
40
|
+
return new Ingestor(params);
|
|
41
|
+
};
|
|
42
|
+
exports.createIngestor = createIngestor;
|
|
43
|
+
|
|
44
|
+
//# sourceMappingURL=Ingestor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"names":["Ingestor","constructor","params","createIngestorResult","getSource","ingest","records","result","record","detail","body","source","name","console","error","log","JSON","stringify","items","item","add","exports","createIngestor"],"sources":["Ingestor.ts"],"sourcesContent":["import type { IIngestor, IIngestorIngestParams, IIngestorResult } from \"./types.js\";\nimport type { IDeployment } from \"~/resolver/deployment/types.js\";\n\nexport interface IIngestorParams {\n createIngestorResult: () => IIngestorResult;\n getSource: (name: string) => IDeployment | null;\n}\n\nexport class Ingestor implements IIngestor {\n private readonly createIngestorResult: () => IIngestorResult;\n private readonly getSource: (name: string) => IDeployment | null;\n\n public constructor(params: IIngestorParams) {\n this.createIngestorResult = params.createIngestorResult;\n this.getSource = params.getSource;\n }\n\n public async ingest(params: IIngestorIngestParams): Promise<IIngestorResult> {\n const { records } = params;\n\n const result = this.createIngestorResult();\n\n for (const record of records) {\n const detail = record.body.detail;\n const source = this.getSource(detail.source.name);\n if (!source) {\n console.error(\n `Could not find deployment for SQS Record source: ${detail.source.name}. More info in next log line.`\n );\n console.log(\n JSON.stringify({\n source: detail.source,\n items: detail.items\n })\n );\n continue;\n }\n\n for (const item of detail.items) {\n result.add({\n item,\n source\n });\n }\n }\n\n return result;\n }\n}\n\nexport const createIngestor = (params: IIngestorParams): IIngestor => {\n return new Ingestor(params);\n};\n"],"mappings":";;;;;;AAQO,MAAMA,QAAQ,CAAsB;EAIhCC,WAAWA,CAACC,MAAuB,EAAE;IACxC,IAAI,CAACC,oBAAoB,GAAGD,MAAM,CAACC,oBAAoB;IACvD,IAAI,CAACC,SAAS,GAAGF,MAAM,CAACE,SAAS;EACrC;EAEA,MAAaC,MAAMA,CAACH,MAA6B,EAA4B;IACzE,MAAM;MAAEI;IAAQ,CAAC,GAAGJ,MAAM;IAE1B,MAAMK,MAAM,GAAG,IAAI,CAACJ,oBAAoB,CAAC,CAAC;IAE1C,KAAK,MAAMK,MAAM,IAAIF,OAAO,EAAE;MAC1B,MAAMG,MAAM,GAAGD,MAAM,CAACE,IAAI,CAACD,MAAM;MACjC,MAAME,MAAM,GAAG,IAAI,CAACP,SAAS,CAACK,MAAM,CAACE,MAAM,CAACC,IAAI,CAAC;MACjD,IAAI,CAACD,MAAM,EAAE;QACTE,OAAO,CAACC,KAAK,CACT,oDAAoDL,MAAM,CAACE,MAAM,CAACC,IAAI,+BAC1E,CAAC;QACDC,OAAO,CAACE,GAAG,CACPC,IAAI,CAACC,SAAS,CAAC;UACXN,MAAM,EAAEF,MAAM,CAACE,MAAM;UACrBO,KAAK,EAAET,MAAM,CAACS;QAClB,CAAC,CACL,CAAC;QACD;MACJ;MAEA,KAAK,MAAMC,IAAI,IAAIV,MAAM,CAACS,KAAK,EAAE;QAC7BX,MAAM,CAACa,GAAG,CAAC;UACPD,IAAI;UACJR;QACJ,CAAC,CAAC;MACN;IACJ;IAEA,OAAOJ,MAAM;EACjB;AACJ;AAACc,OAAA,CAAArB,QAAA,GAAAA,QAAA;AAEM,MAAMsB,cAAc,GAAIpB,MAAuB,IAAgB;EAClE,OAAO,IAAIF,QAAQ,CAACE,MAAM,CAAC;AAC/B,CAAC;AAACmB,OAAA,CAAAC,cAAA,GAAAA,cAAA","ignoreList":[]}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { IIngestorResult, IIngestorResultAddParams, IIngestorResultItem } from "./types.js";
|
|
2
|
+
export declare class IngestorResult implements IIngestorResult {
|
|
3
|
+
private readonly items;
|
|
4
|
+
add(params: IIngestorResultAddParams): void;
|
|
5
|
+
getItems(): IIngestorResultItem[];
|
|
6
|
+
}
|
|
7
|
+
export declare const createIngestorResult: () => IIngestorResult;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
Object.defineProperty(exports, "__esModule", {
|
|
4
|
+
value: true
|
|
5
|
+
});
|
|
6
|
+
exports.createIngestorResult = exports.IngestorResult = void 0;
|
|
7
|
+
class IngestorResult {
|
|
8
|
+
items = new Map();
|
|
9
|
+
add(params) {
|
|
10
|
+
const {
|
|
11
|
+
item,
|
|
12
|
+
source
|
|
13
|
+
} = params;
|
|
14
|
+
let table;
|
|
15
|
+
try {
|
|
16
|
+
table = source.getTable(item.tableType);
|
|
17
|
+
} catch (ex) {
|
|
18
|
+
console.error(`Could not find table for SQS Record source: ${item.tableName} / ${item.tableType}. More info in next log line.`);
|
|
19
|
+
console.log(JSON.stringify(item));
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
const command = item.command === "delete" ? "delete" : "put";
|
|
23
|
+
const key = `PK:${item.PK}#SK:${item.SK}#cmd:${command}#source:${source.name}`;
|
|
24
|
+
if (this.items.has(key)) {
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
this.items.set(key, {
|
|
28
|
+
PK: item.PK,
|
|
29
|
+
SK: item.SK,
|
|
30
|
+
source,
|
|
31
|
+
command,
|
|
32
|
+
table
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
getItems() {
|
|
36
|
+
return Array.from(this.items.values());
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
exports.IngestorResult = IngestorResult;
|
|
40
|
+
const createIngestorResult = () => {
|
|
41
|
+
return new IngestorResult();
|
|
42
|
+
};
|
|
43
|
+
exports.createIngestorResult = createIngestorResult;
|
|
44
|
+
|
|
45
|
+
//# sourceMappingURL=IngestorResult.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"names":["IngestorResult","items","Map","add","params","item","source","table","getTable","tableType","ex","console","error","tableName","log","JSON","stringify","command","key","PK","SK","name","has","set","getItems","Array","from","values","exports","createIngestorResult"],"sources":["IngestorResult.ts"],"sourcesContent":["import type { IIngestorResult, IIngestorResultAddParams, IIngestorResultItem } from \"./types.js\";\nimport type { ITable } from \"~/sync/types.js\";\n\nexport class IngestorResult implements IIngestorResult {\n private readonly items: Map<string, IIngestorResultItem> = new Map();\n\n public add(params: IIngestorResultAddParams): void {\n const { item, source } = params;\n let table: ITable;\n try {\n table = source.getTable(item.tableType);\n } catch (ex) {\n console.error(\n `Could not find table for SQS Record source: ${item.tableName} / ${item.tableType}. More info in next log line.`\n );\n console.log(JSON.stringify(item));\n return;\n }\n\n const command = item.command === \"delete\" ? \"delete\" : \"put\";\n\n const key = `PK:${item.PK}#SK:${item.SK}#cmd:${command}#source:${source.name}`;\n\n if (this.items.has(key)) {\n return;\n }\n\n this.items.set(key, {\n PK: item.PK,\n SK: item.SK,\n source,\n command,\n table\n });\n }\n\n public getItems(): IIngestorResultItem[] {\n return Array.from(this.items.values());\n }\n}\n\nexport const createIngestorResult = (): IIngestorResult => {\n return new IngestorResult();\n};\n"],"mappings":";;;;;;AAGO,MAAMA,cAAc,CAA4B;EAClCC,KAAK,GAAqC,IAAIC,GAAG,CAAC,CAAC;EAE7DC,GAAGA,CAACC,MAAgC,EAAQ;IAC/C,MAAM;MAAEC,IAAI;MAAEC;IAAO,CAAC,GAAGF,MAAM;IAC/B,IAAIG,KAAa;IACjB,IAAI;MACAA,KAAK,GAAGD,MAAM,CAACE,QAAQ,CAACH,IAAI,CAACI,SAAS,CAAC;IAC3C,CAAC,CAAC,OAAOC,EAAE,EAAE;MACTC,OAAO,CAACC,KAAK,CACT,+CAA+CP,IAAI,CAACQ,SAAS,MAAMR,IAAI,CAACI,SAAS,+BACrF,CAAC;MACDE,OAAO,CAACG,GAAG,CAACC,IAAI,CAACC,SAAS,CAACX,IAAI,CAAC,CAAC;MACjC;IACJ;IAEA,MAAMY,OAAO,GAAGZ,IAAI,CAACY,OAAO,KAAK,QAAQ,GAAG,QAAQ,GAAG,KAAK;IAE5D,MAAMC,GAAG,GAAG,MAAMb,IAAI,CAACc,EAAE,OAAOd,IAAI,CAACe,EAAE,QAAQH,OAAO,WAAWX,MAAM,CAACe,IAAI,EAAE;IAE9E,IAAI,IAAI,CAACpB,KAAK,CAACqB,GAAG,CAACJ,GAAG,CAAC,EAAE;MACrB;IACJ;IAEA,IAAI,CAACjB,KAAK,CAACsB,GAAG,CAACL,GAAG,EAAE;MAChBC,EAAE,EAAEd,IAAI,CAACc,EAAE;MACXC,EAAE,EAAEf,IAAI,CAACe,EAAE;MACXd,MAAM;MACNW,OAAO;MACPV;IACJ,CAAC,CAAC;EACN;EAEOiB,QAAQA,CAAA,EAA0B;IACrC,OAAOC,KAAK,CAACC,IAAI,CAAC,IAAI,CAACzB,KAAK,CAAC0B,MAAM,CAAC,CAAC,CAAC;EAC1C;AACJ;AAACC,OAAA,CAAA5B,cAAA,GAAAA,cAAA;AAEM,MAAM6B,oBAAoB,GAAGA,CAAA,KAAuB;EACvD,OAAO,IAAI7B,cAAc,CAAC,CAAC;AAC/B,CAAC;AAAC4B,OAAA,CAAAC,oBAAA,GAAAA,oBAAA","ignoreList":[]}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { CommandType } from "../../../types.js";
|
|
2
|
+
import type { IResolverSQSRecord } from "../abstractions/ResolverRecord.js";
|
|
3
|
+
import type { IDetailItem } from "../../../sync/handler/types.js";
|
|
4
|
+
import type { ITable } from "../../../sync/types.js";
|
|
5
|
+
import type { IDeployment } from "../../deployment/types.js";
|
|
6
|
+
export interface IIngestorIngestParams {
|
|
7
|
+
records: IResolverSQSRecord[];
|
|
8
|
+
}
|
|
9
|
+
export interface IIngestorResultItem {
|
|
10
|
+
PK: string;
|
|
11
|
+
SK: string;
|
|
12
|
+
command: CommandType;
|
|
13
|
+
table: ITable;
|
|
14
|
+
source: IDeployment;
|
|
15
|
+
}
|
|
16
|
+
export interface IIngestorResultAddParams {
|
|
17
|
+
item: IDetailItem;
|
|
18
|
+
source: IDeployment;
|
|
19
|
+
}
|
|
20
|
+
export interface IIngestorResult {
|
|
21
|
+
getItems(): IIngestorResultItem[];
|
|
22
|
+
add(params: IIngestorResultAddParams): void;
|
|
23
|
+
}
|
|
24
|
+
export interface IIngestor {
|
|
25
|
+
ingest(params: IIngestorIngestParams): Promise<IIngestorResult>;
|
|
26
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"names":[],"sources":["types.ts"],"sourcesContent":["import type { CommandType } from \"~/types.js\";\nimport type { IResolverSQSRecord } from \"~/resolver/app/abstractions/ResolverRecord.js\";\nimport type { IDetailItem } from \"~/sync/handler/types.js\";\nimport type { ITable } from \"~/sync/types.js\";\nimport type { IDeployment } from \"~/resolver/deployment/types.js\";\n\nexport interface IIngestorIngestParams {\n records: IResolverSQSRecord[];\n}\n\nexport interface IIngestorResultItem {\n PK: string;\n SK: string;\n command: CommandType;\n table: ITable;\n source: IDeployment;\n}\n\nexport interface IIngestorResultAddParams {\n item: IDetailItem;\n source: IDeployment;\n}\n\nexport interface IIngestorResult {\n getItems(): IIngestorResultItem[];\n add(params: IIngestorResultAddParams): void;\n}\n\nexport interface IIngestor {\n ingest(params: IIngestorIngestParams): Promise<IIngestorResult>;\n}\n"],"mappings":"","ignoreList":[]}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { IStoreItem, IStorer, IStorerExecParams } from "./types";
|
|
2
|
+
import type { IDeployment } from "../../deployment/types.js";
|
|
3
|
+
import type { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb/index.js";
|
|
4
|
+
import type { CommandType } from "../../../types";
|
|
5
|
+
import type { ITable } from "../../../sync/types.js";
|
|
6
|
+
export interface IStorerParamsCreateDocumentClientCallable {
|
|
7
|
+
(deployment: Pick<IDeployment, "region">): Pick<DynamoDBDocument, "send">;
|
|
8
|
+
}
|
|
9
|
+
export interface IStorerOnErrorParams {
|
|
10
|
+
command: CommandType;
|
|
11
|
+
item: IStoreItem;
|
|
12
|
+
table: ITable;
|
|
13
|
+
error: Error;
|
|
14
|
+
}
|
|
15
|
+
export interface IStorerOnErrorCb {
|
|
16
|
+
(params: IStorerOnErrorParams): Promise<void>;
|
|
17
|
+
}
|
|
18
|
+
export interface ISingleStorerAfterEachParams {
|
|
19
|
+
command: CommandType;
|
|
20
|
+
target: IDeployment;
|
|
21
|
+
source: IDeployment;
|
|
22
|
+
table: ITable;
|
|
23
|
+
item: IStoreItem;
|
|
24
|
+
}
|
|
25
|
+
export interface IStorerAfterEachCb {
|
|
26
|
+
(params: ISingleStorerAfterEachParams): Promise<void>;
|
|
27
|
+
}
|
|
28
|
+
export interface IStorerParams {
|
|
29
|
+
maxRetries?: number;
|
|
30
|
+
retryDelay?: number;
|
|
31
|
+
createDocumentClient: IStorerParamsCreateDocumentClientCallable;
|
|
32
|
+
onError?: IStorerOnErrorCb;
|
|
33
|
+
afterEach?: IStorerAfterEachCb;
|
|
34
|
+
}
|
|
35
|
+
export declare class Storer implements IStorer {
|
|
36
|
+
private readonly maxRetries;
|
|
37
|
+
private readonly retryDelay;
|
|
38
|
+
private readonly createDocumentClient;
|
|
39
|
+
private readonly afterEach?;
|
|
40
|
+
private readonly onError?;
|
|
41
|
+
constructor(params: IStorerParams);
|
|
42
|
+
store(params: IStorerExecParams): Promise<void>;
|
|
43
|
+
}
|
|
44
|
+
export declare const createStorer: (params: IStorerParams) => IStorer;
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
Object.defineProperty(exports, "__esModule", {
|
|
4
|
+
value: true
|
|
5
|
+
});
|
|
6
|
+
exports.createStorer = exports.Storer = void 0;
|
|
7
|
+
var _index = require("@webiny/aws-sdk/client-dynamodb/index.js");
|
|
8
|
+
var _Retry = require("../utils/Retry");
|
|
9
|
+
var _utils = require("@webiny/utils");
|
|
10
|
+
class Storer {
|
|
11
|
+
constructor(params) {
|
|
12
|
+
this.maxRetries = params.maxRetries || 10;
|
|
13
|
+
this.retryDelay = params.retryDelay || 1000;
|
|
14
|
+
this.createDocumentClient = params.createDocumentClient;
|
|
15
|
+
this.afterEach = params.afterEach;
|
|
16
|
+
this.onError = params.onError;
|
|
17
|
+
}
|
|
18
|
+
async store(params) {
|
|
19
|
+
const {
|
|
20
|
+
deployment,
|
|
21
|
+
bundle,
|
|
22
|
+
table,
|
|
23
|
+
command,
|
|
24
|
+
items
|
|
25
|
+
} = params;
|
|
26
|
+
if (items.length === 0) {
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
const client = this.createDocumentClient({
|
|
30
|
+
region: deployment.region
|
|
31
|
+
});
|
|
32
|
+
const retry = (0, _Retry.createRetry)({
|
|
33
|
+
maxRetries: this.maxRetries,
|
|
34
|
+
retryDelay: this.retryDelay
|
|
35
|
+
});
|
|
36
|
+
for (const item of items) {
|
|
37
|
+
const result = await retry.retry(async () => {
|
|
38
|
+
switch (command) {
|
|
39
|
+
case "delete":
|
|
40
|
+
return await client.send(new _index.DeleteCommand({
|
|
41
|
+
TableName: table.name,
|
|
42
|
+
Key: {
|
|
43
|
+
PK: item.PK,
|
|
44
|
+
SK: item.SK
|
|
45
|
+
}
|
|
46
|
+
}));
|
|
47
|
+
case "put":
|
|
48
|
+
return await client.send(new _index.PutCommand({
|
|
49
|
+
TableName: table.name,
|
|
50
|
+
Item: item
|
|
51
|
+
}));
|
|
52
|
+
default:
|
|
53
|
+
console.error(`Unsupported command type: ${command}`);
|
|
54
|
+
return null;
|
|
55
|
+
}
|
|
56
|
+
}, {
|
|
57
|
+
onFail: async error => {
|
|
58
|
+
console.error("Error executing batch write command.");
|
|
59
|
+
console.log((0, _utils.convertException)(error));
|
|
60
|
+
if (!this.onError) {
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
try {
|
|
64
|
+
await this.onError({
|
|
65
|
+
item,
|
|
66
|
+
command,
|
|
67
|
+
table,
|
|
68
|
+
error
|
|
69
|
+
});
|
|
70
|
+
} catch (ex) {
|
|
71
|
+
console.error(`Error in onError callback for command: ${command}`);
|
|
72
|
+
console.log({
|
|
73
|
+
original: (0, _utils.convertException)(error),
|
|
74
|
+
error: (0, _utils.convertException)(ex)
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
});
|
|
79
|
+
if (!result || !this.afterEach) {
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
try {
|
|
83
|
+
await this.afterEach({
|
|
84
|
+
table,
|
|
85
|
+
command,
|
|
86
|
+
item,
|
|
87
|
+
source: bundle.source,
|
|
88
|
+
target: deployment
|
|
89
|
+
});
|
|
90
|
+
} catch (ex) {
|
|
91
|
+
console.error(`Error in afterEach callback for command: ${command}`);
|
|
92
|
+
console.log((0, _utils.convertException)(ex));
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
exports.Storer = Storer;
|
|
98
|
+
const createStorer = params => {
|
|
99
|
+
return new Storer(params);
|
|
100
|
+
};
|
|
101
|
+
exports.createStorer = createStorer;
|
|
102
|
+
|
|
103
|
+
//# sourceMappingURL=Storer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"names":["_index","require","_Retry","_utils","Storer","constructor","params","maxRetries","retryDelay","createDocumentClient","afterEach","onError","store","deployment","bundle","table","command","items","length","client","region","retry","createRetry","item","result","send","DeleteCommand","TableName","name","Key","PK","SK","PutCommand","Item","console","error","onFail","log","convertException","ex","original","source","target","exports","createStorer"],"sources":["Storer.ts"],"sourcesContent":["import type { IStoreItem, IStorer, IStorerExecParams } from \"./types\";\nimport type { IDeployment } from \"~/resolver/deployment/types.js\";\nimport type {\n DeleteCommandOutput,\n DynamoDBDocument,\n PutCommandOutput\n} from \"@webiny/aws-sdk/client-dynamodb/index.js\";\nimport { DeleteCommand, PutCommand } from \"@webiny/aws-sdk/client-dynamodb/index.js\";\nimport type { CommandType } from \"~/types\";\nimport type { ITable } from \"~/sync/types.js\";\nimport { createRetry } from \"../utils/Retry\";\nimport { convertException } from \"@webiny/utils\";\n\nexport interface IStorerParamsCreateDocumentClientCallable {\n (deployment: Pick<IDeployment, \"region\">): Pick<DynamoDBDocument, \"send\">;\n}\n\nexport interface IStorerOnErrorParams {\n command: CommandType;\n item: IStoreItem;\n table: ITable;\n error: Error;\n}\n\nexport interface IStorerOnErrorCb {\n (params: IStorerOnErrorParams): Promise<void>;\n}\n\nexport interface ISingleStorerAfterEachParams {\n command: CommandType;\n target: IDeployment;\n source: IDeployment;\n table: ITable;\n item: IStoreItem;\n}\n\nexport interface IStorerAfterEachCb {\n (params: ISingleStorerAfterEachParams): Promise<void>;\n}\n\nexport interface IStorerParams {\n maxRetries?: number;\n retryDelay?: number;\n createDocumentClient: IStorerParamsCreateDocumentClientCallable;\n onError?: IStorerOnErrorCb;\n afterEach?: IStorerAfterEachCb;\n}\n\nexport class Storer implements IStorer {\n private readonly maxRetries: number;\n private readonly retryDelay: number;\n private readonly createDocumentClient: IStorerParamsCreateDocumentClientCallable;\n private readonly afterEach?: IStorerAfterEachCb;\n private readonly onError?: IStorerOnErrorCb;\n\n public constructor(params: IStorerParams) {\n this.maxRetries = params.maxRetries || 10;\n this.retryDelay = params.retryDelay || 1000;\n this.createDocumentClient = params.createDocumentClient;\n this.afterEach = params.afterEach;\n this.onError = params.onError;\n }\n\n public async store(params: IStorerExecParams): Promise<void> {\n const { deployment, bundle, table, command, items } = params;\n if (items.length === 0) {\n return;\n }\n const client = this.createDocumentClient({\n region: deployment.region\n });\n\n const retry = createRetry({\n maxRetries: this.maxRetries,\n retryDelay: this.retryDelay\n });\n\n for (const item of items) {\n const result = await retry.retry(\n async (): Promise<DeleteCommandOutput | PutCommandOutput | null> => {\n switch (command) {\n case \"delete\":\n return await client.send(\n new DeleteCommand({\n TableName: table.name,\n Key: {\n PK: item.PK,\n SK: item.SK\n }\n })\n );\n case \"put\":\n return await client.send(\n new PutCommand({\n TableName: table.name,\n Item: item\n })\n );\n\n default:\n console.error(`Unsupported command type: ${command}`);\n return null;\n }\n },\n {\n onFail: async error => {\n console.error(\"Error executing batch write command.\");\n console.log(convertException(error));\n if (!this.onError) {\n return;\n }\n try {\n await this.onError({\n item,\n command,\n table,\n error\n });\n } catch (ex) {\n console.error(`Error in onError callback for command: ${command}`);\n console.log({\n original: convertException(error),\n error: convertException(ex)\n });\n }\n }\n }\n );\n if (!result || !this.afterEach) {\n continue;\n }\n try {\n await this.afterEach({\n table,\n command,\n item,\n source: bundle.source,\n target: deployment\n });\n } catch (ex) {\n console.error(`Error in afterEach callback for command: ${command}`);\n console.log(convertException(ex));\n }\n }\n }\n}\n\nexport const createStorer = (params: IStorerParams): IStorer => {\n return new Storer(params);\n};\n"],"mappings":";;;;;;AAOA,IAAAA,MAAA,GAAAC,OAAA;AAGA,IAAAC,MAAA,GAAAD,OAAA;AACA,IAAAE,MAAA,GAAAF,OAAA;AAqCO,MAAMG,MAAM,CAAoB;EAO5BC,WAAWA,CAACC,MAAqB,EAAE;IACtC,IAAI,CAACC,UAAU,GAAGD,MAAM,CAACC,UAAU,IAAI,EAAE;IACzC,IAAI,CAACC,UAAU,GAAGF,MAAM,CAACE,UAAU,IAAI,IAAI;IAC3C,IAAI,CAACC,oBAAoB,GAAGH,MAAM,CAACG,oBAAoB;IACvD,IAAI,CAACC,SAAS,GAAGJ,MAAM,CAACI,SAAS;IACjC,IAAI,CAACC,OAAO,GAAGL,MAAM,CAACK,OAAO;EACjC;EAEA,MAAaC,KAAKA,CAACN,MAAyB,EAAiB;IACzD,MAAM;MAAEO,UAAU;MAAEC,MAAM;MAAEC,KAAK;MAAEC,OAAO;MAAEC;IAAM,CAAC,GAAGX,MAAM;IAC5D,IAAIW,KAAK,CAACC,MAAM,KAAK,CAAC,EAAE;MACpB;IACJ;IACA,MAAMC,MAAM,GAAG,IAAI,CAACV,oBAAoB,CAAC;MACrCW,MAAM,EAAEP,UAAU,CAACO;IACvB,CAAC,CAAC;IAEF,MAAMC,KAAK,GAAG,IAAAC,kBAAW,EAAC;MACtBf,UAAU,EAAE,IAAI,CAACA,UAAU;MAC3BC,UAAU,EAAE,IAAI,CAACA;IACrB,CAAC,CAAC;IAEF,KAAK,MAAMe,IAAI,IAAIN,KAAK,EAAE;MACtB,MAAMO,MAAM,GAAG,MAAMH,KAAK,CAACA,KAAK,CAC5B,YAAoE;QAChE,QAAQL,OAAO;UACX,KAAK,QAAQ;YACT,OAAO,MAAMG,MAAM,CAACM,IAAI,CACpB,IAAIC,oBAAa,CAAC;cACdC,SAAS,EAAEZ,KAAK,CAACa,IAAI;cACrBC,GAAG,EAAE;gBACDC,EAAE,EAAEP,IAAI,CAACO,EAAE;gBACXC,EAAE,EAAER,IAAI,CAACQ;cACb;YACJ,CAAC,CACL,CAAC;UACL,KAAK,KAAK;YACN,OAAO,MAAMZ,MAAM,CAACM,IAAI,CACpB,IAAIO,iBAAU,CAAC;cACXL,SAAS,EAAEZ,KAAK,CAACa,IAAI;cACrBK,IAAI,EAAEV;YACV,CAAC,CACL,CAAC;UAEL;YACIW,OAAO,CAACC,KAAK,CAAC,6BAA6BnB,OAAO,EAAE,CAAC;YACrD,OAAO,IAAI;QACnB;MACJ,CAAC,EACD;QACIoB,MAAM,EAAE,MAAMD,KAAK,IAAI;UACnBD,OAAO,CAACC,KAAK,CAAC,sCAAsC,CAAC;UACrDD,OAAO,CAACG,GAAG,CAAC,IAAAC,uBAAgB,EAACH,KAAK,CAAC,CAAC;UACpC,IAAI,CAAC,IAAI,CAACxB,OAAO,EAAE;YACf;UACJ;UACA,IAAI;YACA,MAAM,IAAI,CAACA,OAAO,CAAC;cACfY,IAAI;cACJP,OAAO;cACPD,KAAK;cACLoB;YACJ,CAAC,CAAC;UACN,CAAC,CAAC,OAAOI,EAAE,EAAE;YACTL,OAAO,CAACC,KAAK,CAAC,0CAA0CnB,OAAO,EAAE,CAAC;YAClEkB,OAAO,CAACG,GAAG,CAAC;cACRG,QAAQ,EAAE,IAAAF,uBAAgB,EAACH,KAAK,CAAC;cACjCA,KAAK,EAAE,IAAAG,uBAAgB,EAACC,EAAE;YAC9B,CAAC,CAAC;UACN;QACJ;MACJ,CACJ,CAAC;MACD,IAAI,CAACf,MAAM,IAAI,CAAC,IAAI,CAACd,SAAS,EAAE;QAC5B;MACJ;MACA,IAAI;QACA,MAAM,IAAI,CAACA,SAAS,CAAC;UACjBK,KAAK;UACLC,OAAO;UACPO,IAAI;UACJkB,MAAM,EAAE3B,MAAM,CAAC2B,MAAM;UACrBC,MAAM,EAAE7B;QACZ,CAAC,CAAC;MACN,CAAC,CAAC,OAAO0B,EAAE,EAAE;QACTL,OAAO,CAACC,KAAK,CAAC,4CAA4CnB,OAAO,EAAE,CAAC;QACpEkB,OAAO,CAACG,GAAG,CAAC,IAAAC,uBAAgB,EAACC,EAAE,CAAC,CAAC;MACrC;IACJ;EACJ;AACJ;AAACI,OAAA,CAAAvC,MAAA,GAAAA,MAAA;AAEM,MAAMwC,YAAY,GAAItC,MAAqB,IAAc;EAC5D,OAAO,IAAIF,MAAM,CAACE,MAAM,CAAC;AAC7B,CAAC;AAACqC,OAAA,CAAAC,YAAA,GAAAA,YAAA","ignoreList":[]}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { IDeployment } from "../../deployment/types.js";
|
|
2
|
+
import type { ITable } from "../../../sync/types.js";
|
|
3
|
+
import type { CommandType } from "../../../types.js";
|
|
4
|
+
import type { GenericRecord } from "@webiny/api/types.js";
|
|
5
|
+
import type { IBundle } from "../bundler/types";
|
|
6
|
+
export type StoreItemPossibleValues = string | number | boolean | undefined | null | GenericRecord | StoreItemPossibleValues[];
|
|
7
|
+
export interface IStoreItem {
|
|
8
|
+
PK: string;
|
|
9
|
+
SK: string;
|
|
10
|
+
[key: string]: StoreItemPossibleValues;
|
|
11
|
+
}
|
|
12
|
+
export interface IStorerExecParams {
|
|
13
|
+
deployment: IDeployment;
|
|
14
|
+
bundle: IBundle;
|
|
15
|
+
command: CommandType;
|
|
16
|
+
table: ITable;
|
|
17
|
+
items: IStoreItem[];
|
|
18
|
+
}
|
|
19
|
+
export interface IStorer {
|
|
20
|
+
store(params: IStorerExecParams): Promise<void>;
|
|
21
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"names":[],"sources":["types.ts"],"sourcesContent":["import type { IDeployment } from \"~/resolver/deployment/types.js\";\nimport type { ITable } from \"~/sync/types.js\";\nimport type { CommandType } from \"~/types.js\";\nimport type { GenericRecord } from \"@webiny/api/types.js\";\nimport type { IBundle } from \"../bundler/types\";\n\nexport type StoreItemPossibleValues =\n | string\n | number\n | boolean\n | undefined\n | null\n | GenericRecord\n | StoreItemPossibleValues[];\n\nexport interface IStoreItem {\n PK: string;\n SK: string;\n [key: string]: StoreItemPossibleValues;\n}\n\nexport interface IStorerExecParams {\n deployment: IDeployment;\n bundle: IBundle;\n command: CommandType;\n table: ITable;\n items: IStoreItem[];\n}\n\nexport interface IStorer {\n store(params: IStorerExecParams): Promise<void>;\n}\n"],"mappings":"","ignoreList":[]}
|