@mbc-cqrs-serverless/cli 0.1.73-beta.0 → 0.1.75-beta.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mbc-cqrs-serverless/cli",
-  "version": "0.1.73-beta.0",
+  "version": "0.1.75-beta.0",
   "description": "a CLI to get started with MBC CQRS serverless framework",
   "keywords": [
     "mbc",
@@ -58,5 +58,5 @@
     "@faker-js/faker": "^8.3.1",
     "copyfiles": "^2.4.1"
   },
-  "gitHead": "3399562c3c03284ea21e96a40cd61fd0a68977cb"
+  "gitHead": "a9c91c6ab9ca15188d7b7075989e90048248721c"
 }

package/templates/infra/libs/distributed-map.ts

@@ -0,0 +1,105 @@
+import {
+  IChainable,
+  JsonPath,
+  ProcessorConfig,
+  ProcessorMode,
+  Map as SfnMap,
+} from 'aws-cdk-lib/aws-stepfunctions'
+
+export type DistributedMapS3Parameter =
+  | {
+      readonly 'Bucket.$': string
+      readonly 'Key.$': string
+    }
+  | {
+      readonly Bucket: JsonPath | string
+      readonly Key: JsonPath | string
+    }
+
+export interface DistributedMapItemReader {
+  readonly Resource:
+    | 'arn:aws:states:::s3:getObject'
+    | 'arn:aws:states:::s3:listObjectsV2'
+  readonly ReaderConfig: {
+    readonly InputType: 'CSV' | 'JSON' | 'MANIFEST'
+    readonly CSVHeaderLocation?: 'FIRST_ROW' | 'GIVEN'
+    readonly CSVHeaders?: string[]
+    readonly MaxItems?: number
+  }
+  readonly Parameters: DistributedMapS3Parameter
+}
+
+export interface DistributedMapResultWriter {
+  readonly Resource: 'arn:aws:states:::s3:putObject'
+  readonly Parameters: DistributedMapS3Parameter
+}
+
+export interface DistributedMapItemBatcher {
+  readonly MaxItemsPerBatch?: number
+  readonly MaxItemsPerBatchPath?: string
+  readonly MaxInputBytesPerBatch?: number
+  readonly MaxInputBytesPerBatchPath?: number
+  readonly BatchInput?: Readonly<Record<string, JsonPath | string>>
+}
+
+export class DistributedMap extends SfnMap {
+  public itemReader?: DistributedMapItemReader
+  public resultWriter?: DistributedMapResultWriter
+  public itemBatcher?: DistributedMapItemBatcher
+  declare public itemSelector?: Readonly<Record<string, JsonPath | string>>
+  public label?: string
+
+  public override toStateJson(): object {
+    const mapStateJson = super.toStateJson()
+    return {
+      ...mapStateJson,
+      ItemReader: this.itemReader,
+      ResultWriter: this.resultWriter,
+      ItemBatcher: this.itemBatcher,
+      ItemSelector: this.itemSelector,
+      Label: this.label,
+    }
+  }
+
+  public itemProcessor(
+    processor: IChainable,
+    config: ProcessorConfig = {},
+  ): DistributedMap {
+    super.itemProcessor(processor, {
+      ...config,
+      mode: ProcessorMode.DISTRIBUTED,
+    })
+    return this
+  }
+
+  public setLabel(label: string): DistributedMap {
+    this.label = label
+    return this
+  }
+
+  public setItemSelector(
+    itemSelector: Readonly<Record<string, JsonPath | string>>,
+  ): DistributedMap {
+    this.itemSelector = itemSelector
+    return this
+  }
+
+  public setItemBatcher(
+    itemBatcher: DistributedMapItemBatcher,
+  ): DistributedMap {
+    this.itemBatcher = itemBatcher
+    return this
+  }
+
+  public setResultWriter(
+    resultWriter: DistributedMapResultWriter,
+  ): DistributedMap {
+    this.resultWriter = resultWriter
+    return this
+  }
+
+  public setItemReader(itemReader: DistributedMapItemReader): DistributedMap {
+    this.itemReader = itemReader
+    return this
+  }
+}

package/templates/infra/libs/infra-stack.ts

@@ -20,6 +20,7 @@ import {
 } from '../config'
 import { Config } from '../config/type'
 import { buildApp } from './build-app'
+import { DistributedMap } from './distributed-map'
 
 export interface InfraStackProps extends cdk.StackProps {
   config: Config
@@ -115,6 +116,10 @@ export class InfraStack extends cdk.Stack {
       },
     })
 
+    const importActionSqs = new cdk.aws_sqs.Queue(this, 'import-action-sqs', {
+      queueName: prefix + 'import-action-queue',
+    })
+
     const subTaskStatusSqs = new cdk.aws_sqs.Queue(
       this,
       'sub-task-status-sqs',
@@ -163,6 +168,17 @@ export class InfraStack extends cdk.Stack {
         },
       }),
     )
+
+    mainSns.addSubscription(
+      new cdk.aws_sns_subscriptions.SqsSubscription(importActionSqs, {
+        rawMessageDelivery: true,
+        filterPolicy: {
+          action: cdk.aws_sns.SubscriptionFilter.stringFilter({
+            allowlist: ['import-execute'],
+          }),
+        },
+      }),
+    )
     // host zone
     const hostedZone = cdk.aws_route53.HostedZone.fromHostedZoneAttributes(
       this,
@@ -353,6 +369,18 @@ export class InfraStack extends cdk.Stack {
       arnFormat: cdk.ArnFormat.COLON_RESOURCE_NAME,
     })
 
+    const importCsvStateMachineName = prefix + 'import-csv-handler'
+
+    const importCsvSfnArn = cdk.Arn.format({
+      partition: 'aws',
+      region: this.region,
+      account: this.account,
+      service: 'states',
+      resource: 'stateMachine',
+      resourceName: importCsvStateMachineName,
+      arnFormat: cdk.ArnFormat.COLON_RESOURCE_NAME,
+    })
+
     // Lambda api ( arm64 )
     const execEnv = {
       NODE_OPTIONS: '--enable-source-maps',
@@ -365,6 +393,7 @@ export class InfraStack extends cdk.Stack {
       S3_BUCKET_NAME: ddbBucket.bucketName,
       SFN_COMMAND_ARN: commandSfnArn,
       SFN_TASK_ARN: taskSfnArn,
+      SFN_IMPORT_CSV_ARN: importCsvSfnArn,
       SNS_TOPIC_ARN: mainSns.topicArn,
       SNS_ALARM_TOPIC_ARN: alarmSns.topicArn,
       COGNITO_USER_POOL_ID: userPool.userPoolId,
@@ -851,6 +880,65 @@ export class InfraStack extends cdk.Stack {
       },
     )
 
+    // Csv import
+    const csvRowsHandlerState = lambdaInvoke(
+      'csv_rows_handler',
+      null,
+      cdk.aws_stepfunctions.IntegrationPattern.REQUEST_RESPONSE,
+    )
+
+    const sfnImportCsvDefinition = new DistributedMap(this, 'import-csv', {
+      maxConcurrency: 50,
+    })
+      .setLabel('import-csv')
+      .setItemReader({
+        Resource: 'arn:aws:states:::s3:getObject',
+        ReaderConfig: {
+          InputType: 'CSV',
+          CSVHeaderLocation: 'FIRST_ROW',
+        },
+        Parameters: {
+          'Bucket.$': '$.bucket',
+          'Key.$': '$.key',
+        },
+      })
+      .setItemBatcher({
+        MaxInputBytesPerBatch: 10,
+        BatchInput: {
+          'Attributes.$': '$',
+        },
+      })
+      .itemProcessor(csvRowsHandlerState, {
+        executionType: cdk.aws_stepfunctions.ProcessorType.EXPRESS,
+      })
+
+    const sfnImportCsvLogGroup = new cdk.aws_logs.LogGroup(
+      this,
+      'import-csv-handler-sfn-log',
+      {
+        logGroupName: `/aws/vendedlogs/states/${prefix}-import-csv-handler-state-machine-logs`, // Specify a log group name
+        removalPolicy: cdk.RemovalPolicy.DESTROY, // Policy for log group removal
+        retention: cdk.aws_logs.RetentionDays.SIX_MONTHS,
+      },
+    )
+
+    const importCsvStateMachine = new cdk.aws_stepfunctions.StateMachine(
+      this,
+      'import-csv-state-machine',
+      {
+        stateMachineName: importCsvStateMachineName,
+        comment: 'A state machine for import-csv module handler',
+        definitionBody: cdk.aws_stepfunctions.DefinitionBody.fromChainable(
+          sfnImportCsvDefinition,
+        ),
+        tracingEnabled: true,
+        logs: {
+          destination: sfnImportCsvLogGroup,
+          level: cdk.aws_stepfunctions.LogLevel.ALL, // Log level (ALL, ERROR, or FATAL)
+        },
+      },
+    )
+
     // add event sources to lambda event
     lambdaApi.addEventSource(
       new cdk.aws_lambda_event_sources.SqsEventSource(taskSqs, {
@@ -867,8 +955,13 @@ export class InfraStack extends cdk.Stack {
         batchSize: 1,
       }),
     )
+    lambdaApi.addEventSource(
+      new cdk.aws_lambda_event_sources.SqsEventSource(importActionSqs, {
+        batchSize: 1,
+      }),
+    )
     // dynamodb event source
-    const tableNames = ['tasks', 'sample-command']
+    const tableNames = ['tasks', 'sample-command', 'import_tmp']
     for (const tableName of tableNames) {
       const tableDesc = new cdk.custom_resources.AwsCustomResource(
         this,
@@ -926,6 +1019,17 @@ export class InfraStack extends cdk.Stack {
       'cognito-idp:AdminUpdateUserAttributes',
     )
     ddbBucket.grantReadWrite(lambdaApi)
+    ddbBucket.grantRead(importCsvStateMachine)
+    importCsvStateMachine.role?.attachInlinePolicy(
+      new cdk.aws_iam.Policy(this, 'csv-import-map-policy', {
+        statements: [
+          new cdk.aws_iam.PolicyStatement({
+            actions: ['states:StartExecution'],
+            resources: [importCsvSfnArn],
+          }),
+        ],
+      }),
+    )
     publicBucket.grantReadWrite(lambdaApi)
     mainSns.grantPublish(lambdaApi)
     alarmSns.grantPublish(lambdaApi)
@@ -973,6 +1077,11 @@ export class InfraStack extends cdk.Stack {
       resources: [taskSfnArn], // Access to all resources
     })
 
+    const importCsvSfnPolicy = new cdk.aws_iam.PolicyStatement({
+      actions: ['states:*'],
+      resources: [importCsvSfnArn], // Access to all resources
+    })
+
     // Attach the policy to the Lambda function's execution role
     lambdaApi.role?.attachInlinePolicy(
       new cdk.aws_iam.Policy(this, 'lambda-event-sfn-policy', {
@@ -991,6 +1100,16 @@ export class InfraStack extends cdk.Stack {
       }),
     )
 
+    lambdaApi.role?.attachInlinePolicy(
+      new cdk.aws_iam.Policy(
+        this,
+        'lambda-import-csv-sfn-step-function-policy',
+        {
+          statements: [importCsvSfnPolicy.copy()],
+        },
+      ),
+    )
+
     // Attach the policy to the Lambda function's execution role
     lambdaApi.role?.attachInlinePolicy(
       new cdk.aws_iam.Policy(this, 'lambda-ses-policy', {

package/templates/infra-local/elasticmq.conf

@@ -25,6 +25,11 @@ queues {
         delay = 5 seconds
         receiveMessageWait = 0 seconds
     }
+    import-action-queue {
+        defaultVisibilityTimeout = 60 seconds
+        delay = 5 seconds
+        receiveMessageWait = 0 seconds
+    }
 }
 
 aws {

package/templates/infra-local/serverless.yml

@@ -45,6 +45,11 @@ custom:
           rawMessageDelivery: 'true'
           filterPolicy: { 'action': ['sub-task-status'] }
         queue: http://localhost:9324/101010101010/sub-task-status-queue
+      - topic:
+          topicName: CqrsSnsTopic
+          rawMessageDelivery: 'true'
+          filterPolicy: { 'action': ['import-execute'] }
+        queue: http://localhost:9324/101010101010/import-action-queue
       - topic:
           topicName: AlarmSnsTopic
           rawMessageDelivery: 'true'
@@ -157,6 +162,11 @@ functions:
             Fn::GetAtt:
               - NotificationQueue
               - Arn
+      - sqs:
+          arn:
+            Fn::GetAtt:
+              - ImportActionQueue
+              - Arn
       - stream:
           type: dynamodb
           maximumRetryAttempts: 10
@@ -169,6 +179,13 @@ functions:
           arn: ${env:LOCAL_DDB_TASKS_STREAM}
           filterPatterns:
             - eventName: [INSERT]
+      - stream:
+          type: dynamodb
+          maximumRetryAttempts: 10
+          arn: ${env:LOCAL_DDB_IMPORT_TMP_STREAM}
+          filterPatterns:
+            - eventName:
+                - INSERT
 
 stepFunctions:
   # https://goessner.net/articles/JsonPath/index.html
@@ -366,6 +383,48 @@ stepFunctions:
                   End: true
             End: true
             # MaxConcurrency: 2
+    import-csv:
+      name: import-csv
+      definition:
+        Comment: import csv data
+        StartAt: csv_loader
+        States:
+          csv_loader:
+            Type: Task
+            Resource: arn:aws:states:::lambda:invoke
+            Parameters:
+              FunctionName: arn:aws:lambda:ap-northeast-1:101010101010:function:serverless-example-dev-main
+              Payload:
+                input.$: $
+                context.$: $$
+            Retry:
+              - ErrorEquals:
+                  - Lambda.ServiceException
+                  - Lambda.AWSLambdaException
+                  - Lambda.SdkClientException
+                IntervalSeconds: 2
+                MaxAttempts: 5
+                BackoffRate: 2
+            OutputPath: $.Payload[0][0]
+            Next: read_csv
+          read_csv:
+            Type: Task
+            Resource: arn:aws:states:::lambda:invoke
+            Parameters:
+              FunctionName: arn:aws:lambda:ap-northeast-1:101010101010:function:serverless-example-dev-main
+              Payload:
+                input.$: $
+                context.$: $$
+            Retry:
+              - ErrorEquals:
+                  - Lambda.ServiceException
+                  - Lambda.AWSLambdaException
+                  - Lambda.SdkClientException
+                IntervalSeconds: 2
+                MaxAttempts: 5
+                BackoffRate: 2
+            OutputPath: $.Payload[0][0]
+            End: true
 resources:
   Resources:
     TaskActionQueue:
@@ -380,3 +439,7 @@ resources:
       Type: AWS::SQS::Queue
       Properties:
         QueueName: notification-queue
+    ImportActionQueue:
+      Type: AWS::SQS::Queue
+      Properties:
+        QueueName: import-action-queue