@cdklabs/cdk-appmod-catalog-blueprints 1.0.0

package/lib/utilities/data-loader.d.ts

@@ -0,0 +1,173 @@
+import { Duration, RemovalPolicy, CustomResource } from 'aws-cdk-lib';
+import { IVpc, ISecurityGroup } from 'aws-cdk-lib/aws-ec2';
+import { PolicyStatement } from 'aws-cdk-lib/aws-iam';
+import { Function as LambdaFunction } from 'aws-cdk-lib/aws-lambda';
+import { IDatabaseCluster, IDatabaseInstance } from 'aws-cdk-lib/aws-rds';
+import { Bucket } from 'aws-cdk-lib/aws-s3';
+import { BucketDeployment } from 'aws-cdk-lib/aws-s3-deployment';
+import { ISecret } from 'aws-cdk-lib/aws-secretsmanager';
+import { StateMachine } from 'aws-cdk-lib/aws-stepfunctions';
+import { Provider } from 'aws-cdk-lib/custom-resources';
+import { Construct } from 'constructs';
+/**
+ * Supported database engines
+ */
+export declare enum DatabaseEngine {
+    MYSQL = "mysql",
+    POSTGRESQL = "postgresql"
+}
+/**
+ * Supported file types for data loading
+ */
+export declare enum FileType {
+    /** Standard SQL file */
+    SQL = "sql",
+    /** MySQL dump file generated by mysqldump */
+    MYSQLDUMP = "mysqldump",
+    /** PostgreSQL dump file generated by pg_dump */
+    PGDUMP = "pgdump"
+}
+/**
+ * Database connection configuration
+ */
+export interface DatabaseConfig {
+    /** Database engine type */
+    readonly engine: DatabaseEngine;
+    /** Database cluster (for Aurora) */
+    readonly cluster?: IDatabaseCluster;
+    /** Database instance (for RDS) */
+    readonly instance?: IDatabaseInstance;
+    /** Database credentials secret */
+    readonly secret: ISecret;
+    /** Database name to connect to */
+    readonly databaseName: string;
+    /** VPC where the database is located */
+    readonly vpc: IVpc;
+    /** Security group for database access */
+    readonly securityGroup: ISecurityGroup;
+}
+/**
+ * File input configuration
+ */
+export interface FileInput {
+    /** Path to the file (local path or S3 URI) */
+    readonly filePath: string;
+    /** Type of file */
+    readonly fileType: FileType;
+    /** Execution order (lower numbers execute first) */
+    readonly executionOrder?: number;
+    /** Whether to continue on error */
+    readonly continueOnError?: boolean;
+}
+/**
+ * Properties for the DataLoader construct
+ */
+export interface DataLoaderProps {
+    /** Database configuration */
+    readonly databaseConfig: DatabaseConfig;
+    /** List of files to load */
+    readonly fileInputs: FileInput[];
+    /** Optional removal policy for resources (defaults to DESTROY) */
+    readonly removalPolicy?: RemovalPolicy;
+    /** Optional timeout for Lambda function (defaults to 15 minutes) */
+    readonly timeout?: Duration;
+    /** Optional memory size for Lambda function (defaults to 1024 MB) */
+    readonly memorySize?: number;
+}
+/**
+ * DataLoader construct for loading data into Aurora/RDS databases
+ *
+ * This construct provides a simplified solution for loading data from various file formats
+ * (SQL, mysqldump, pg_dump) into MySQL or PostgreSQL databases. It uses S3 for file storage,
+ * Step Functions for orchestration, and Lambda for processing.
+ *
+ * Architecture:
+ * 1. Files are uploaded to S3 bucket
+ * 2. Step Function is triggered with list of S3 keys
+ * 3. Step Function iterates over files in execution order
+ * 4. Lambda function processes each file against the database
+ *
+ * Example usage:
+ * Create a DataLoader with database configuration and file inputs.
+ * The construct will handle uploading files to S3, creating a Step Function
+ * to orchestrate processing, and executing the data loading pipeline.
+ */
+export declare class DataLoader extends Construct {
+    /** The S3 bucket used for storing files */
+    readonly bucket: Bucket;
+    /** The Step Functions state machine for orchestration */
+    readonly stateMachine: StateMachine;
+    /** The Lambda function that processes the data loading */
+    readonly processorFunction: LambdaFunction;
+    /** The bucket deployment for uploading files */
+    bucketDeployment?: BucketDeployment;
+    /** The custom resource provider for triggering state machine execution */
+    readonly customResourceProvider: Provider;
+    /** The custom resource that triggers the state machine */
+    readonly executionTrigger: CustomResource;
+    /** The file inputs configuration */
+    private readonly fileInputs;
+    constructor(scope: Construct, id: string, props: DataLoaderProps);
+    /**
+     * Grants additional IAM permissions to the execution trigger Lambda function
+     * @param statement The IAM policy statement to add
+     */
+    grantExecutionTriggerPermissions(statement: PolicyStatement): void;
+    /**
+     * Validates the construct properties
+     * @param props The DataLoader properties
+     * @private
+     */
+    private _validateProps;
+    /**
+     * Creates the S3 bucket for storing files
+     * @param removalPolicy The removal policy to apply
+     * @returns The created S3 bucket
+     * @private
+     */
+    private _createBucket;
+    /**
+     * Creates the Lambda function for processing data loading
+     * @param props The DataLoader properties
+     * @returns The created Lambda function
+     * @private
+     */
+    private _createProcessorFunction;
+    /**
+     * Creates the Step Functions state machine
+     * @returns The created state machine
+     * @private
+     */
+    private _createStateMachine;
+    /**
+     * Grants necessary permissions to the Lambda function
+     * @param lambdaFunction The Lambda function
+     * @param props The DataLoader properties
+     * @private
+     */
+    private _grantPermissions;
+    /**
+     * Sets up file processing by uploading files to S3
+     * @param props The DataLoader properties
+     * @private
+     */
+    private _setupFileProcessing;
+    /**
+     * Creates a custom resource provider for triggering state machine execution
+     * @returns The custom resource provider
+     * @private
+     */
+    private _createCustomResourceProvider;
+    /**
+     * Creates a custom resource to trigger state machine execution
+     * @returns The custom resource
+     * @private
+     */
+    private _createExecutionTrigger;
+    /**
+     * Gets the ordered file keys for execution
+     * @returns Array of S3 keys in execution order
+     * @private
+     */
+    private _getOrderedFileKeys;
+}