@cdklabs/cdk-appmod-catalog-blueprints 1.0.0

package/lib/utilities/cdk-nag-config.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Shared CDK Nag configuration and suppressions for AppMod Use Case Blueprints
+ *
+ * This module provides common CDK Nag suppressions that can be reused across
+ * different use cases while maintaining security best practices.
+ */
+export interface NagSuppression {
+    id: string;
+    reason: string;
+}
+/**
+ * Common suppressions that apply across multiple use cases
+ */
+export declare const COMMON_NAG_SUPPRESSIONS: NagSuppression[];
+/**
+ * Document processing specific suppressions
+ */
+export declare const DOCUMENT_PROCESSING_NAG_SUPPRESSIONS: NagSuppression[];
+/**
+ * Web application specific suppressions
+ */
+export declare const WEBAPP_NAG_SUPPRESSIONS: NagSuppression[];
+/**
+ * Frontend specific suppressions
+ */
+export declare const FRONTEND_NAG_SUPPRESSIONS: NagSuppression[];
+/**
+ * Development/testing specific suppressions
+ */
+export declare const DEVELOPMENT_NAG_SUPPRESSIONS: NagSuppression[];
+/**
+ * Production specific suppressions (stricter requirements)
+ */
+export declare const PRODUCTION_NAG_SUPPRESSIONS: NagSuppression[];
+/**
+ * Utility function to get suppressions based on use case and environment
+ */
+export declare function getNagSuppressions(useCase: 'document-processing' | 'webapp' | 'frontend' | 'common', environment?: 'development' | 'staging' | 'production'): NagSuppression[];
+/**
+ * Utility function to create custom suppressions for specific scenarios
+ */
+export declare function createCustomSuppression(id: string, reason: string): NagSuppression;

package/lib/utilities/cdk-nag-config.js

@@ -0,0 +1,194 @@
+"use strict";
+/**
+ * Shared CDK Nag configuration and suppressions for AppMod Use Case Blueprints
+ *
+ * This module provides common CDK Nag suppressions that can be reused across
+ * different use cases while maintaining security best practices.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PRODUCTION_NAG_SUPPRESSIONS = exports.DEVELOPMENT_NAG_SUPPRESSIONS = exports.FRONTEND_NAG_SUPPRESSIONS = exports.WEBAPP_NAG_SUPPRESSIONS = exports.DOCUMENT_PROCESSING_NAG_SUPPRESSIONS = exports.COMMON_NAG_SUPPRESSIONS = void 0;
+exports.getNagSuppressions = getNagSuppressions;
+exports.createCustomSuppression = createCustomSuppression;
+/**
+ * Common suppressions that apply across multiple use cases
+ */
+exports.COMMON_NAG_SUPPRESSIONS = [
+    {
+        id: 'AwsSolutions-IAM4',
+        reason: 'AWS managed policies provide appropriate permissions and are maintained by AWS',
+    },
+    {
+        id: 'AwsSolutions-L1',
+        reason: 'Lambda runtime versions are managed at the application deployment level',
+    },
+];
+/**
+ * Document processing specific suppressions
+ */
+exports.DOCUMENT_PROCESSING_NAG_SUPPRESSIONS = [
+    ...exports.COMMON_NAG_SUPPRESSIONS,
+    {
+        id: 'AwsSolutions-S3-1',
+        reason: 'Document processing bucket requires controlled public access for result delivery',
+    },
+    {
+        id: 'AwsSolutions-S3-2',
+        reason: 'Public read access is managed by application-level security controls',
+    },
+    {
+        id: 'AwsSolutions-IAM5',
+        reason: 'Document processing requires broad permissions across multiple AWS services (S3, Textract, SQS, Step Functions)',
+    },
+    {
+        id: 'AwsSolutions-SQS3',
+        reason: 'Dead letter queue configuration is handled by the base document processor',
+    },
+    {
+        id: 'AwsSolutions-SF1',
+        reason: 'Step Functions logging configuration is environment-specific',
+    },
+    {
+        id: 'AwsSolutions-SF2',
+        reason: 'X-Ray tracing is configured based on monitoring requirements',
+    },
+];
+/**
+ * Web application specific suppressions
+ */
+exports.WEBAPP_NAG_SUPPRESSIONS = [
+    ...exports.COMMON_NAG_SUPPRESSIONS,
+    {
+        id: 'AwsSolutions-ECS2',
+        reason: 'Environment variables are managed securely through configuration management',
+    },
+    {
+        id: 'AwsSolutions-ECS4',
+        reason: 'Container insights are enabled based on monitoring requirements and cost considerations',
+    },
+    {
+        id: 'AwsSolutions-IAM5',
+        reason: 'Web application requires broad permissions for ECS, RDS, and other AWS service integration',
+    },
+    {
+        id: 'AwsSolutions-ELB2',
+        reason: 'Load balancer access logging is configured based on compliance requirements',
+    },
+    {
+        id: 'AwsSolutions-RDS2',
+        reason: 'Database encryption is configured based on data sensitivity classification',
+    },
+    {
+        id: 'AwsSolutions-RDS3',
+        reason: 'Multi-AZ configuration is based on availability SLA requirements',
+    },
+    {
+        id: 'AwsSolutions-RDS6',
+        reason: 'Database authentication method is chosen based on application architecture',
+    },
+    {
+        id: 'AwsSolutions-RDS10',
+        reason: 'Database deletion protection is managed through deployment automation',
+    },
+    {
+        id: 'AwsSolutions-RDS11',
+        reason: 'Standard database ports are acceptable within VPC security boundaries',
+    },
+    {
+        id: 'AwsSolutions-SMG4',
+        reason: 'Secrets rotation policy is configured per organizational security standards',
+    },
+];
+/**
+ * Frontend specific suppressions
+ */
+exports.FRONTEND_NAG_SUPPRESSIONS = [
+    ...exports.COMMON_NAG_SUPPRESSIONS,
+    {
+        id: 'AwsSolutions-CFR1',
+        reason: 'CloudFront geo restrictions are configured based on application requirements',
+    },
+    {
+        id: 'AwsSolutions-CFR2',
+        reason: 'CloudFront WAF integration is configured based on security requirements',
+    },
+    {
+        id: 'AwsSolutions-CFR3',
+        reason: 'CloudFront access logging is configured based on compliance requirements',
+    },
+    {
+        id: 'AwsSolutions-CFR4',
+        reason: 'CloudFront viewer protocol policy is set to redirect-to-https for security',
+    },
+    {
+        id: 'AwsSolutions-S1',
+        reason: 'S3 bucket access logging is configured based on compliance requirements',
+    },
+    {
+        id: 'AwsSolutions-S2',
+        reason: 'S3 bucket public access is blocked and access is controlled via CloudFront OAC',
+    },
+    {
+        id: 'AwsSolutions-S3',
+        reason: 'S3 bucket SSL requests only policy is enforced via CloudFront HTTPS redirect',
+    },
+    {
+        id: 'AwsSolutions-S10',
+        reason: 'S3 bucket MFA delete is managed through organizational security policies',
+    },
+];
+/**
+ * Development/testing specific suppressions
+ */
+exports.DEVELOPMENT_NAG_SUPPRESSIONS = [
+    {
+        id: 'AwsSolutions-RDS3',
+        reason: 'Multi-AZ not required for development environments',
+    },
+    {
+        id: 'AwsSolutions-ELB2',
+        reason: 'Access logging not required for development load balancers',
+    },
+    {
+        id: 'AwsSolutions-CFR3',
+        reason: 'CloudFront access logging not required for development environments',
+    },
+];
+/**
+ * Production specific suppressions (stricter requirements)
+ */
+exports.PRODUCTION_NAG_SUPPRESSIONS = [
+// Production environments should have minimal suppressions
+// Most security controls should be enabled
+];
+/**
+ * Utility function to get suppressions based on use case and environment
+ */
+function getNagSuppressions(useCase, environment = 'production') {
+    let suppressions = [];
+    switch (useCase) {
+        case 'document-processing':
+            suppressions = [...exports.DOCUMENT_PROCESSING_NAG_SUPPRESSIONS];
+            break;
+        case 'webapp':
+            suppressions = [...exports.WEBAPP_NAG_SUPPRESSIONS];
+            break;
+        case 'frontend':
+            suppressions = [...exports.FRONTEND_NAG_SUPPRESSIONS];
+            break;
+        case 'common':
+            suppressions = [...exports.COMMON_NAG_SUPPRESSIONS];
+            break;
+    }
+    // Add environment-specific suppressions
+    if (environment === 'development') {
+        suppressions = [...suppressions, ...exports.DEVELOPMENT_NAG_SUPPRESSIONS];
+    }
+    return suppressions;
+}
+/**
+ * Utility function to create custom suppressions for specific scenarios
+ */
+function createCustomSuppression(id, reason) {
+    return { id, reason };
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY2RrLW5hZy1jb25maWcuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi91c2UtY2FzZXMvdXRpbGl0aWVzL2Nkay1uYWctY29uZmlnLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7QUFBQTs7Ozs7R0FLRzs7O0FBdUtILGdEQTJCQztBQUtELDBEQUVDO0FBbE1EOztHQUVHO0FBQ1UsUUFBQSx1QkFBdUIsR0FBcUI7SUFDdkQ7UUFDRSxFQUFFLEVBQUUsbUJBQW1CO1FBQ3ZCLE1BQU0sRUFBRSxnRkFBZ0Y7S0FDekY7SUFDRDtRQUNFLEVBQUUsRUFBRSxpQkFBaUI7UUFDckIsTUFBTSxFQUFFLHlFQUF5RTtLQUNsRjtDQUNGLENBQUM7QUFFRjs7R0FFRztBQUNVLFFBQUEsb0NBQW9DLEdBQXFCO0lBQ3BFLEdBQUcsK0JBQXVCO0lBQzFCO1FBQ0UsRUFBRSxFQUFFLG1CQUFtQjtRQUN2QixNQUFNLEVBQUUsa0ZBQWtGO0tBQzNGO0lBQ0Q7UUFDRSxFQUFFLEVBQUUsbUJBQW1CO1FBQ3ZCLE1BQU0sRUFBRSxzRUFBc0U7S0FDL0U7SUFDRDtRQUNFLEVBQUUsRUFBRSxtQkFBbUI7UUFDdkIsTUFBTSxFQUFFLGlIQUFpSDtLQUMxSDtJQUNEO1FBQ0UsRUFBRSxFQUFFLG1CQUFtQjtRQUN2QixNQUFNLEVBQUUsMkVBQTJFO0tBQ3BGO0lBQ0Q7UUFDRSxFQUFFLEVBQUUsa0JBQWtCO1FBQ3RCLE1BQU0sRUFBRSw4REFBOEQ7S0FDdkU7SUFDRDtRQUNFLEVBQUUsRUFBRSxrQkFBa0I7UUFDdEIsTUFBTSxFQUFFLDhEQUE4RDtLQUN2RTtDQUNGLENBQUM7QUFFRjs7R0FFRztBQUNVLFFBQUEsdUJBQXVCLEdBQXFCO0lBQ3ZELEdBQUcsK0JBQXVCO0lBQzFCO1FBQ0UsRUFBRSxFQUFFLG1CQUFtQjtRQUN2QixNQUFNLEVBQUUsNkVBQTZFO0tBQ3RGO0lBQ0Q7UUFDRSxFQUFFLEVBQUUsbUJBQW1CO1FBQ3ZCLE1BQU0sRUFBRSx5RkFBeUY7S0FDbEc7SUFDRDtRQUNFLEVBQUUsRUFBRSxtQkFBbUI7UUFDdkIsTUFBTSxFQUFFLDRGQUE0RjtLQUNyRztJQUNEO1FBQ0UsRUFBRSxFQUFFLG1CQUFtQjtRQUN2QixNQUFNLEVBQUUsNkVBQTZFO0tBQ3RGO0lBQ0Q7UUFDRSxFQUFFLEVBQUUsbUJBQW1CO1FBQ3ZCLE1BQU0sRUFBRSw0RUFBNEU7S0FDckY7SUFDRDtRQUNFLEVBQUUsRUFBRSxtQkFBbUI7UUFDdkIsTUFBTSxFQUFFLGtFQUFrRTtLQUMzRTtJQUNEO1FBQ0UsRUFBRSxFQUFFLG1CQUFtQjtRQUN2QixNQUFNLEVBQUUsNEVBQTRFO0tBQ3JGO0lBQ0Q7UUFDRSxFQUFFLEVBQUUsb0JBQW9CO1FBQ3hCLE1BQU0sRUFBRSx1RUFBdUU7S0FDaEY7SUFDRDtRQUNFLEVBQUUsRUFBRSxvQkFBb0I7UUFDeEIsTUFBTSxFQUFFLHVFQUF1RTtLQUNoRjtJQUNEO1FBQ0UsRUFBRSxFQUFFLG1CQUFtQjtRQUN2QixNQUFNLEVBQUUsNkVBQTZFO0tBQ3RGO0NBQ0YsQ0FBQztBQUVGOztHQUVHO0FBQ1UsUUFBQSx5QkFBeUIsR0FBcUI7SUFDekQsR0FBRywrQkFBdUI7SUFDMUI7UUFDRSxFQUFFLEVBQUUsbUJBQW1CO1FBQ3ZCLE1BQU0sRUFBRSw4RUFBOEU7S0FDdkY7SUFDRDtRQUNFLEVBQUUsRUFBRSxtQkFBbUI7UUFDdkIsTUFBTSxFQUFFLHlFQUF5RTtLQUNsRjtJQUNEO1FBQ0UsRUFBRSxFQUFFLG1CQUFtQjtRQUN2QixNQUFNLEVBQUUsMEVBQTBFO0tBQ25GO0lBQ0Q7UUFDRSxFQUFFLEVBQUUsbUJBQW1CO1FBQ3ZCLE1BQU0sRUFBRSw0RUFBNEU7S0FDckY7SUFDRDtRQUNFLEVBQUUsRUFBRSxpQkFBaUI7UUFDckIsTUFBTSxFQUFFLHlFQUF5RTtLQUNsRjtJQUNEO1FBQ0UsRUFBRSxFQUFFLGlCQUFpQjtRQUNyQixNQUFNLEVBQUUsZ0ZBQWdGO0tBQ3pGO0lBQ0Q7UUFDRSxFQUFFLEVBQUUsaUJBQWlCO1FBQ3JCLE1BQU0sRUFBRSw4RUFBOEU7S0FDdkY7SUFDRDtRQUNFLEVBQUUsRUFBRSxrQkFBa0I7UUFDdEIsTUFBTSxFQUFFLDBFQUEwRTtLQUNuRjtDQUNGLENBQUM7QUFFRjs7R0FFRztBQUNVLFFBQUEsNEJBQTRCLEdBQXFCO0lBQzVEO1FBQ0UsRUFBRSxFQUFFLG1CQUFtQjtRQUN2QixNQUFNLEVBQUUsb0RBQW9EO0tBQzdEO0lBQ0Q7UUFDRSxFQUFFLEVBQUUsbUJBQW1CO1FBQ3ZCLE1BQU0sRUFBRSw0REFBNEQ7S0FDckU7SUFDRDtRQUNFLEVBQUUsRUFBRSxtQkFBbUI7UUFDdkIsTUFBTSxFQUFFLHFFQUFxRTtLQUM5RTtDQUNGLENBQUM7QUFFRjs7R0FFRztBQUNVLFFBQUEsMkJBQTJCLEdBQXFCO0FBQzNELDJEQUEyRDtBQUMzRCwyQ0FBMkM7Q0FDNUMsQ0FBQztBQUVGOztHQUVHO0FBQ0gsU0FBZ0Isa0JBQWtCLENBQ2hDLE9BQWlFLEVBQ2pFLGNBQXdELFlBQVk7SUFFcEUsSUFBSSxZQUFZLEdBQXFCLEVBQUUsQ0FBQztJQUV4QyxRQUFRLE9BQU8sRUFBRSxDQUFDO1FBQ2hCLEtBQUsscUJBQXFCO1lBQ3hCLFlBQVksR0FBRyxDQUFDLEdBQUcsNENBQW9DLENBQUMsQ0FBQztZQUN6RCxNQUFNO1FBQ1IsS0FBSyxRQUFRO1lBQ1gsWUFBWSxHQUFHLENBQUMsR0FBRywrQkFBdUIsQ0FBQyxDQUFDO1lBQzVDLE1BQU07UUFDUixLQUFLLFVBQVU7WUFDYixZQUFZLEdBQUcsQ0FBQyxHQUFHLGlDQUF5QixDQUFDLENBQUM7WUFDOUMsTUFBTTtRQUNSLEtBQUssUUFBUTtZQUNYLFlBQVksR0FBRyxDQUFDLEdBQUcsK0JBQXVCLENBQUMsQ0FBQztZQUM1QyxNQUFNO0lBQ1YsQ0FBQztJQUVELHdDQUF3QztJQUN4QyxJQUFJLFdBQVcsS0FBSyxhQUFhLEVBQUUsQ0FBQztRQUNsQyxZQUFZLEdBQUcsQ0FBQyxHQUFHLFlBQVksRUFBRSxHQUFHLG9DQUE0QixDQUFDLENBQUM7SUFDcEUsQ0FBQztJQUVELE9BQU8sWUFBWSxDQUFDO0FBQ3RCLENBQUM7QUFFRDs7R0FFRztBQUNILFNBQWdCLHVCQUF1QixDQUFDLEVBQVUsRUFBRSxNQUFjO0lBQ2hFLE9BQU8sRUFBRSxFQUFFLEVBQUUsTUFBTSxFQUFFLENBQUM7QUFDeEIsQ0FBQyIsInNvdXJjZXNDb250ZW50IjpbIi8qKlxuICogU2hhcmVkIENESyBOYWcgY29uZmlndXJhdGlvbiBhbmQgc3VwcHJlc3Npb25zIGZvciBBcHBNb2QgVXNlIENhc2UgQmx1ZXByaW50c1xuICpcbiAqIFRoaXMgbW9kdWxlIHByb3ZpZGVzIGNvbW1vbiBDREsgTmFnIHN1cHByZXNzaW9ucyB0aGF0IGNhbiBiZSByZXVzZWQgYWNyb3NzXG4gKiBkaWZmZXJlbnQgdXNlIGNhc2VzIHdoaWxlIG1haW50YWluaW5nIHNlY3VyaXR5IGJlc3QgcHJhY3RpY2VzLlxuICovXG5cbmV4cG9ydCBpbnRlcmZhY2UgTmFnU3VwcHJlc3Npb24ge1xuICBpZDogc3RyaW5nO1xuICByZWFzb246IHN0cmluZztcbn1cblxuLyoqXG4gKiBDb21tb24gc3VwcHJlc3Npb25zIHRoYXQgYXBwbHkgYWNyb3NzIG11bHRpcGxlIHVzZSBjYXNlc1xuICovXG5leHBvcnQgY29uc3QgQ09NTU9OX05BR19TVVBQUkVTU0lPTlM6IE5hZ1N1cHByZXNzaW9uW10gPSBbXG4gIHtcbiAgICBpZDogJ0F3c1NvbHV0aW9ucy1JQU00JyxcbiAgICByZWFzb246ICdBV1MgbWFuYWdlZCBwb2xpY2llcyBwcm92aWRlIGFwcHJvcHJpYXRlIHBlcm1pc3Npb25zIGFuZCBhcmUgbWFpbnRhaW5lZCBieSBBV1MnLFxuICB9LFxuICB7XG4gICAgaWQ6ICdBd3NTb2x1dGlvbnMtTDEnLFxuICAgIHJlYXNvbjogJ0xhbWJkYSBydW50aW1lIHZlcnNpb25zIGFyZSBtYW5hZ2VkIGF0IHRoZSBhcHBsaWNhdGlvbiBkZXBsb3ltZW50IGxldmVsJyxcbiAgfSxcbl07XG5cbi8qKlxuICogRG9jdW1lbnQgcHJvY2Vzc2luZyBzcGVjaWZpYyBzdXBwcmVzc2lvbnNcbiAqL1xuZXhwb3J0IGNvbnN0IERPQ1VNRU5UX1BST0NFU1NJTkdfTkFHX1NVUFBSRVNTSU9OUzogTmFnU3VwcHJlc3Npb25bXSA9IFtcbiAgLi4uQ09NTU9OX05BR19TVVBQUkVTU0lPTlMsXG4gIHtcbiAgICBpZDogJ0F3c1NvbHV0aW9ucy1TMy0xJyxcbiAgICByZWFzb246ICdEb2N1bWVudCBwcm9jZXNzaW5nIGJ1Y2tldCByZXF1aXJlcyBjb250cm9sbGVkIHB1YmxpYyBhY2Nlc3MgZm9yIHJlc3VsdCBkZWxpdmVyeScsXG4gIH0sXG4gIHtcbiAgICBpZDogJ0F3c1NvbHV0aW9ucy1TMy0yJyxcbiAgICByZWFzb246ICdQdWJsaWMgcmVhZCBhY2Nlc3MgaXMgbWFuYWdlZCBieSBhcHBsaWNhdGlvbi1sZXZlbCBzZWN1cml0eSBjb250cm9scycsXG4gIH0sXG4gIHtcbiAgICBpZDogJ0F3c1NvbHV0aW9ucy1JQU01JyxcbiAgICByZWFzb246ICdEb2N1bWVudCBwcm9jZXNzaW5nIHJlcXVpcmVzIGJyb2FkIHBlcm1pc3Npb25zIGFjcm9zcyBtdWx0aXBsZSBBV1Mgc2VydmljZXMgKFMzLCBUZXh0cmFjdCwgU1FTLCBTdGVwIEZ1bmN0aW9ucyknLFxuICB9LFxuICB7XG4gICAgaWQ6ICdBd3NTb2x1dGlvbnMtU1FTMycsXG4gICAgcmVhc29uOiAnRGVhZCBsZXR0ZXIgcXVldWUgY29uZmlndXJhdGlvbiBpcyBoYW5kbGVkIGJ5IHRoZSBiYXNlIGRvY3VtZW50IHByb2Nlc3NvcicsXG4gIH0sXG4gIHtcbiAgICBpZDogJ0F3c1NvbHV0aW9ucy1TRjEnLFxuICAgIHJlYXNvbjogJ1N0ZXAgRnVuY3Rpb25zIGxvZ2dpbmcgY29uZmlndXJhdGlvbiBpcyBlbnZpcm9ubWVudC1zcGVjaWZpYycsXG4gIH0sXG4gIHtcbiAgICBpZDogJ0F3c1NvbHV0aW9ucy1TRjInLFxuICAgIHJlYXNvbjogJ1gtUmF5IHRyYWNpbmcgaXMgY29uZmlndXJlZCBiYXNlZCBvbiBtb25pdG9yaW5nIHJlcXVpcmVtZW50cycsXG4gIH0sXG5dO1xuXG4vKipcbiAqIFdlYiBhcHBsaWNhdGlvbiBzcGVjaWZpYyBzdXBwcmVzc2lvbnNcbiAqL1xuZXhwb3J0IGNvbnN0IFdFQkFQUF9OQUdfU1VQUFJFU1NJT05TOiBOYWdTdXBwcmVzc2lvbltdID0gW1xuICAuLi5DT01NT05fTkFHX1NVUFBSRVNTSU9OUyxcbiAge1xuICAgIGlkOiAnQXdzU29sdXRpb25zLUVDUzInLFxuICAgIHJlYXNvbjogJ0Vudmlyb25tZW50IHZhcmlhYmxlcyBhcmUgbWFuYWdlZCBzZWN1cmVseSB0aHJvdWdoIGNvbmZpZ3VyYXRpb24gbWFuYWdlbWVudCcsXG4gIH0sXG4gIHtcbiAgICBpZDogJ0F3c1NvbHV0aW9ucy1FQ1M0JyxcbiAgICByZWFzb246ICdDb250YWluZXIgaW5zaWdodHMgYXJlIGVuYWJsZWQgYmFzZWQgb24gbW9uaXRvcmluZyByZXF1aXJlbWVudHMgYW5kIGNvc3QgY29uc2lkZXJhdGlvbnMnLFxuICB9LFxuICB7XG4gICAgaWQ6ICdBd3NTb2x1dGlvbnMtSUFNNScsXG4gICAgcmVhc29uOiAnV2ViIGFwcGxpY2F0aW9uIHJlcXVpcmVzIGJyb2FkIHBlcm1pc3Npb25zIGZvciBFQ1MsIFJEUywgYW5kIG90aGVyIEFXUyBzZXJ2aWNlIGludGVncmF0aW9uJyxcbiAgfSxcbiAge1xuICAgIGlkOiAnQXdzU29sdXRpb25zLUVMQjInLFxuICAgIHJlYXNvbjogJ0xvYWQgYmFsYW5jZXIgYWNjZXNzIGxvZ2dpbmcgaXMgY29uZmlndXJlZCBiYXNlZCBvbiBjb21wbGlhbmNlIHJlcXVpcmVtZW50cycsXG4gIH0sXG4gIHtcbiAgICBpZDogJ0F3c1NvbHV0aW9ucy1SRFMyJyxcbiAgICByZWFzb246ICdEYXRhYmFzZSBlbmNyeXB0aW9uIGlzIGNvbmZpZ3VyZWQgYmFzZWQgb24gZGF0YSBzZW5zaXRpdml0eSBjbGFzc2lmaWNhdGlvbicsXG4gIH0sXG4gIHtcbiAgICBpZDogJ0F3c1NvbHV0aW9ucy1SRFMzJyxcbiAgICByZWFzb246ICdNdWx0aS1BWiBjb25maWd1cmF0aW9uIGlzIGJhc2VkIG9uIGF2YWlsYWJpbGl0eSBTTEEgcmVxdWlyZW1lbnRzJyxcbiAgfSxcbiAge1xuICAgIGlkOiAnQXdzU29sdXRpb25zLVJEUzYnLFxuICAgIHJlYXNvbjogJ0RhdGFiYXNlIGF1dGhlbnRpY2F0aW9uIG1ldGhvZCBpcyBjaG9zZW4gYmFzZWQgb24gYXBwbGljYXRpb24gYXJjaGl0ZWN0dXJlJyxcbiAgfSxcbiAge1xuICAgIGlkOiAnQXdzU29sdXRpb25zLVJEUzEwJyxcbiAgICByZWFzb246ICdEYXRhYmFzZSBkZWxldGlvbiBwcm90ZWN0aW9uIGlzIG1hbmFnZWQgdGhyb3VnaCBkZXBsb3ltZW50IGF1dG9tYXRpb24nLFxuICB9LFxuICB7XG4gICAgaWQ6ICdBd3NTb2x1dGlvbnMtUkRTMTEnLFxuICAgIHJlYXNvbjogJ1N0YW5kYXJkIGRhdGFiYXNlIHBvcnRzIGFyZSBhY2NlcHRhYmxlIHdpdGhpbiBWUEMgc2VjdXJpdHkgYm91bmRhcmllcycsXG4gIH0sXG4gIHtcbiAgICBpZDogJ0F3c1NvbHV0aW9ucy1TTUc0JyxcbiAgICByZWFzb246ICdTZWNyZXRzIHJvdGF0aW9uIHBvbGljeSBpcyBjb25maWd1cmVkIHBlciBvcmdhbml6YXRpb25hbCBzZWN1cml0eSBzdGFuZGFyZHMnLFxuICB9LFxuXTtcblxuLyoqXG4gKiBGcm9udGVuZCBzcGVjaWZpYyBzdXBwcmVzc2lvbnNcbiAqL1xuZXhwb3J0IGNvbnN0IEZST05URU5EX05BR19TVVBQUkVTU0lPTlM6IE5hZ1N1cHByZXNzaW9uW10gPSBbXG4gIC4uLkNPTU1PTl9OQUdfU1VQUFJFU1NJT05TLFxuICB7XG4gICAgaWQ6ICdBd3NTb2x1dGlvbnMtQ0ZSMScsXG4gICAgcmVhc29uOiAnQ2xvdWRGcm9udCBnZW8gcmVzdHJpY3Rpb25zIGFyZSBjb25maWd1cmVkIGJhc2VkIG9uIGFwcGxpY2F0aW9uIHJlcXVpcmVtZW50cycsXG4gIH0sXG4gIHtcbiAgICBpZDogJ0F3c1NvbHV0aW9ucy1DRlIyJyxcbiAgICByZWFzb246ICdDbG91ZEZyb250IFdBRiBpbnRlZ3JhdGlvbiBpcyBjb25maWd1cmVkIGJhc2VkIG9uIHNlY3VyaXR5IHJlcXVpcmVtZW50cycsXG4gIH0sXG4gIHtcbiAgICBpZDogJ0F3c1NvbHV0aW9ucy1DRlIzJyxcbiAgICByZWFzb246ICdDbG91ZEZyb250IGFjY2VzcyBsb2dnaW5nIGlzIGNvbmZpZ3VyZWQgYmFzZWQgb24gY29tcGxpYW5jZSByZXF1aXJlbWVudHMnLFxuICB9LFxuICB7XG4gICAgaWQ6ICdBd3NTb2x1dGlvbnMtQ0ZSNCcsXG4gICAgcmVhc29uOiAnQ2xvdWRGcm9udCB2aWV3ZXIgcHJvdG9jb2wgcG9saWN5IGlzIHNldCB0byByZWRpcmVjdC10by1odHRwcyBmb3Igc2VjdXJpdHknLFxuICB9LFxuICB7XG4gICAgaWQ6ICdBd3NTb2x1dGlvbnMtUzEnLFxuICAgIHJlYXNvbjogJ1MzIGJ1Y2tldCBhY2Nlc3MgbG9nZ2luZyBpcyBjb25maWd1cmVkIGJhc2VkIG9uIGNvbXBsaWFuY2UgcmVxdWlyZW1lbnRzJyxcbiAgfSxcbiAge1xuICAgIGlkOiAnQXdzU29sdXRpb25zLVMyJyxcbiAgICByZWFzb246ICdTMyBidWNrZXQgcHVibGljIGFjY2VzcyBpcyBibG9ja2VkIGFuZCBhY2Nlc3MgaXMgY29udHJvbGxlZCB2aWEgQ2xvdWRGcm9udCBPQUMnLFxuICB9LFxuICB7XG4gICAgaWQ6ICdBd3NTb2x1dGlvbnMtUzMnLFxuICAgIHJlYXNvbjogJ1MzIGJ1Y2tldCBTU0wgcmVxdWVzdHMgb25seSBwb2xpY3kgaXMgZW5mb3JjZWQgdmlhIENsb3VkRnJvbnQgSFRUUFMgcmVkaXJlY3QnLFxuICB9LFxuICB7XG4gICAgaWQ6ICdBd3NTb2x1dGlvbnMtUzEwJyxcbiAgICByZWFzb246ICdTMyBidWNrZXQgTUZBIGRlbGV0ZSBpcyBtYW5hZ2VkIHRocm91Z2ggb3JnYW5pemF0aW9uYWwgc2VjdXJpdHkgcG9saWNpZXMnLFxuICB9LFxuXTtcblxuLyoqXG4gKiBEZXZlbG9wbWVudC90ZXN0aW5nIHNwZWNpZmljIHN1cHByZXNzaW9uc1xuICovXG5leHBvcnQgY29uc3QgREVWRUxPUE1FTlRfTkFHX1NVUFBSRVNTSU9OUzogTmFnU3VwcHJlc3Npb25bXSA9IFtcbiAge1xuICAgIGlkOiAnQXdzU29sdXRpb25zLVJEUzMnLFxuICAgIHJlYXNvbjogJ011bHRpLUFaIG5vdCByZXF1aXJlZCBmb3IgZGV2ZWxvcG1lbnQgZW52aXJvbm1lbnRzJyxcbiAgfSxcbiAge1xuICAgIGlkOiAnQXdzU29sdXRpb25zLUVMQjInLFxuICAgIHJlYXNvbjogJ0FjY2VzcyBsb2dnaW5nIG5vdCByZXF1aXJlZCBmb3IgZGV2ZWxvcG1lbnQgbG9hZCBiYWxhbmNlcnMnLFxuICB9LFxuICB7XG4gICAgaWQ6ICdBd3NTb2x1dGlvbnMtQ0ZSMycsXG4gICAgcmVhc29uOiAnQ2xvdWRGcm9udCBhY2Nlc3MgbG9nZ2luZyBub3QgcmVxdWlyZWQgZm9yIGRldmVsb3BtZW50IGVudmlyb25tZW50cycsXG4gIH0sXG5dO1xuXG4vKipcbiAqIFByb2R1Y3Rpb24gc3BlY2lmaWMgc3VwcHJlc3Npb25zIChzdHJpY3RlciByZXF1aXJlbWVudHMpXG4gKi9cbmV4cG9ydCBjb25zdCBQUk9EVUNUSU9OX05BR19TVVBQUkVTU0lPTlM6IE5hZ1N1cHByZXNzaW9uW10gPSBbXG4gIC8vIFByb2R1Y3Rpb24gZW52aXJvbm1lbnRzIHNob3VsZCBoYXZlIG1pbmltYWwgc3VwcHJlc3Npb25zXG4gIC8vIE1vc3Qgc2VjdXJpdHkgY29udHJvbHMgc2hvdWxkIGJlIGVuYWJsZWRcbl07XG5cbi8qKlxuICogVXRpbGl0eSBmdW5jdGlvbiB0byBnZXQgc3VwcHJlc3Npb25zIGJhc2VkIG9uIHVzZSBjYXNlIGFuZCBlbnZpcm9ubWVudFxuICovXG5leHBvcnQgZnVuY3Rpb24gZ2V0TmFnU3VwcHJlc3Npb25zKFxuICB1c2VDYXNlOiAnZG9jdW1lbnQtcHJvY2Vzc2luZycgfCAnd2ViYXBwJyB8ICdmcm9udGVuZCcgfCAnY29tbW9uJyxcbiAgZW52aXJvbm1lbnQ6ICdkZXZlbG9wbWVudCcgfCAnc3RhZ2luZycgfCAncHJvZHVjdGlvbicgPSAncHJvZHVjdGlvbicsXG4pOiBOYWdTdXBwcmVzc2lvbltdIHtcbiAgbGV0IHN1cHByZXNzaW9uczogTmFnU3VwcHJlc3Npb25bXSA9IFtdO1xuXG4gIHN3aXRjaCAodXNlQ2FzZSkge1xuICAgIGNhc2UgJ2RvY3VtZW50LXByb2Nlc3NpbmcnOlxuICAgICAgc3VwcHJlc3Npb25zID0gWy4uLkRPQ1VNRU5UX1BST0NFU1NJTkdfTkFHX1NVUFBSRVNTSU9OU107XG4gICAgICBicmVhaztcbiAgICBjYXNlICd3ZWJhcHAnOlxuICAgICAgc3VwcHJlc3Npb25zID0gWy4uLldFQkFQUF9OQUdfU1VQUFJFU1NJT05TXTtcbiAgICAgIGJyZWFrO1xuICAgIGNhc2UgJ2Zyb250ZW5kJzpcbiAgICAgIHN1cHByZXNzaW9ucyA9IFsuLi5GUk9OVEVORF9OQUdfU1VQUFJFU1NJT05TXTtcbiAgICAgIGJyZWFrO1xuICAgIGNhc2UgJ2NvbW1vbic6XG4gICAgICBzdXBwcmVzc2lvbnMgPSBbLi4uQ09NTU9OX05BR19TVVBQUkVTU0lPTlNdO1xuICAgICAgYnJlYWs7XG4gIH1cblxuICAvLyBBZGQgZW52aXJvbm1lbnQtc3BlY2lmaWMgc3VwcHJlc3Npb25zXG4gIGlmIChlbnZpcm9ubWVudCA9PT0gJ2RldmVsb3BtZW50Jykge1xuICAgIHN1cHByZXNzaW9ucyA9IFsuLi5zdXBwcmVzc2lvbnMsIC4uLkRFVkVMT1BNRU5UX05BR19TVVBQUkVTU0lPTlNdO1xuICB9XG5cbiAgcmV0dXJuIHN1cHByZXNzaW9ucztcbn1cblxuLyoqXG4gKiBVdGlsaXR5IGZ1bmN0aW9uIHRvIGNyZWF0ZSBjdXN0b20gc3VwcHJlc3Npb25zIGZvciBzcGVjaWZpYyBzY2VuYXJpb3NcbiAqL1xuZXhwb3J0IGZ1bmN0aW9uIGNyZWF0ZUN1c3RvbVN1cHByZXNzaW9uKGlkOiBzdHJpbmcsIHJlYXNvbjogc3RyaW5nKTogTmFnU3VwcHJlc3Npb24ge1xuICByZXR1cm4geyBpZCwgcmVhc29uIH07XG59XG4iXX0=

package/lib/utilities/data-loader-lambda/index.py

@@ -0,0 +1,282 @@
+#!/usr/bin/env python3
+"""
+DataLoader Lambda Function
+
+This Lambda function processes individual files from S3 and loads them into the database.
+It's called by Step Functions for each file that needs to be processed.
+"""
+
+import json
+import os
+import logging
+import boto3
+import pymysql
+import psycopg2
+from typing import Dict, Any, Optional
+import tempfile
+import re
+
+# Configure logging
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+
+# Initialize AWS clients
+s3_client = boto3.client('s3')
+secrets_client = boto3.client('secretsmanager')
+
+class DatabaseConnection:
+    """Handles database connections for MySQL and PostgreSQL"""
+    
+    def __init__(self, engine: str, connection_params: Dict[str, Any]):
+        self.engine = engine.lower()
+        self.connection_params = connection_params
+        self.connection = None
+    
+    def connect(self):
+        """Establish database connection"""
+        try:
+            if self.engine == 'mysql':
+                self.connection = pymysql.connect(
+                    host=self.connection_params['host'],
+                    port=self.connection_params.get('port', 3306),
+                    user=self.connection_params['username'],
+                    password=self.connection_params['password'],
+                    database=self.connection_params['database'],
+                    charset='utf8mb4',
+                    autocommit=False
+                )
+            elif self.engine == 'postgresql':
+                self.connection = psycopg2.connect(
+                    host=self.connection_params['host'],
+                    port=self.connection_params.get('port', 5432),
+                    user=self.connection_params['username'],
+                    password=self.connection_params['password'],
+                    database=self.connection_params['database']
+                )
+                self.connection.autocommit = False
+            else:
+                raise ValueError(f"Unsupported database engine: {self.engine}")
+            
+            logger.info(f"Successfully connected to {self.engine} database")
+            
+        except Exception as e:
+            logger.error(f"Failed to connect to database: {str(e)}")
+            raise
+    
+    def execute_sql_file(self, file_content: str) -> Dict[str, Any]:
+        """Execute SQL content from file against the database as a single operation"""
+        if not self.connection:
+            raise RuntimeError("Database connection not established")
+        
+        results = {
+            'success': True,
+            'file_executed': True,
+            'errors': []
+        }
+        
+        try:            
+            if not file_content.strip():
+                logger.info("No SQL content to execute after cleaning")
+                return results
+            
+            logger.info("Executing entire SQL file content as a single transaction")
+            
+            # Execute the entire file content at once using the appropriate method for each database
+            if self.engine == 'postgresql':
+                self._execute_postgresql_file(file_content)
+            elif self.engine == 'mysql':
+                self._execute_mysql_file(file_content)
+            
+            logger.info("Successfully executed SQL file content")
+            
+        except Exception as e:
+            logger.error(f"Failed to execute SQL file: {str(e)}")
+            results['success'] = False
+            results['file_executed'] = False
+            results['errors'].append(str(e))
+            raise
+        
+        return results
+    
+    def _execute_postgresql_file(self, sql_content: str):
+        """Execute PostgreSQL file content as a single operation"""
+        cursor = self.connection.cursor()
+        try:
+            # PostgreSQL can handle multiple statements in a single execute call
+            cursor.execute(sql_content)
+            self.connection.commit()
+        except Exception as e:
+            self.connection.rollback()
+            raise
+        finally:
+            cursor.close()
+    
+    def _execute_mysql_file(self, sql_content: str):
+        """Execute MySQL file content as a single transaction"""
+        cursor = self.connection.cursor()
+        try:
+            # For MySQL, we need to split statements but execute them all in one transaction
+            statements = self._split_sql_statements(sql_content)
+            
+            # Execute all statements in a single transaction
+            for statement in statements:
+                statement = statement.strip()
+                if statement and not statement.startswith('--'):
+                    cursor.execute(statement)
+            
+            # Commit all changes as one transaction
+            self.connection.commit()
+        except Exception as e:
+            self.connection.rollback()
+            raise
+        finally:
+            cursor.close()
+    
+    def _split_sql_statements(self, sql_content: str) -> list:
+        """Split SQL content into individual statements"""
+        # Remove comments
+        sql_content = re.sub(r'--.*?\n', '\n', sql_content)
+        sql_content = re.sub(r'/\*.*?\*/', '', sql_content, flags=re.DOTALL)
+        
+        # Split by semicolon, but be careful with strings and functions
+        statements = []
+        current_statement = ""
+        in_string = False
+        string_char = None
+        
+        i = 0
+        while i < len(sql_content):
+            char = sql_content[i]
+            
+            if not in_string and char in ["'", '"']:
+                in_string = True
+                string_char = char
+            elif in_string and char == string_char:
+                # Check if it's escaped
+                if i > 0 and sql_content[i-1] != '\\':
+                    in_string = False
+                    string_char = None
+            elif not in_string and char == ';':
+                current_statement += char
+                if current_statement.strip():
+                    statements.append(current_statement.strip())
+                current_statement = ""
+                i += 1
+                continue
+            
+            current_statement += char
+            i += 1
+        
+        # Add the last statement if it doesn't end with semicolon
+        if current_statement.strip():
+            statements.append(current_statement.strip())
+        
+        return statements
+    
+    def close(self):
+        """Close database connection"""
+        if self.connection:
+            self.connection.close()
+            logger.info("Database connection closed")
+
+def get_database_credentials(secret_arn: str) -> Dict[str, Any]:
+    """Retrieve database credentials from AWS Secrets Manager"""
+    try:
+        response = secrets_client.get_secret_value(SecretId=secret_arn)
+        secret_data = json.loads(response['SecretString'])
+        
+        return {
+            'host': secret_data.get('host'),
+            'port': secret_data.get('port'),
+            'username': secret_data.get('username'),
+            'password': secret_data.get('password'),
+            'database': os.environ.get('DATABASE_NAME')
+        }
+    
+    except Exception as e:
+        logger.error(f"Failed to retrieve database credentials: {str(e)}")
+        raise
+
+def download_file_from_s3(bucket_name: str, s3_key: str) -> str:
+    """Download file from S3 and return its content"""
+    try:
+        logger.info(f"Downloading file s3://{bucket_name}/{s3_key}")
+        
+        # Create temporary file
+        temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.sql')
+        temp_file_path = temp_file.name
+        temp_file.close()
+        
+        # Download file
+        s3_client.download_file(bucket_name, s3_key, temp_file_path)
+        
+        # Read file content
+        with open(temp_file_path, 'r', encoding='utf-8') as f:
+            content = f.read()
+        
+        # Clean up temporary file
+        os.unlink(temp_file_path)
+        
+        logger.info(f"Successfully downloaded and read file {s3_key}")
+        return content
+        
+    except Exception as e:
+        logger.error(f"Failed to download file {s3_key}: {str(e)}")
+        raise
+
+def handler(event, context):
+    """Lambda function handler"""
+    logger.info(f"Processing event: {json.dumps(event)}")
+    
+    try:
+        # Extract parameters from event
+        s3_key = event.get('s3Key')
+        bucket_name = event.get('bucketName') or os.environ.get('S3_BUCKET')
+        
+        if not s3_key:
+            raise ValueError("s3Key is required in the event payload")
+        
+        if not bucket_name:
+            raise ValueError("bucketName is required in the event payload or S3_BUCKET environment variable")
+        
+        logger.info(f"Processing file: s3://{bucket_name}/{s3_key}")
+        
+        # Download file content from S3
+        file_content = download_file_from_s3(bucket_name, s3_key)
+        
+        # Get database credentials
+        secret_arn = os.environ['DATABASE_SECRET_ARN']
+        db_credentials = get_database_credentials(secret_arn)
+        
+        # Create database connection
+        engine = os.environ['DATABASE_ENGINE']
+        db_connection = DatabaseConnection(engine, db_credentials)
+        db_connection.connect()
+        
+        try:
+            # Execute the SQL file
+            result = db_connection.execute_sql_file(file_content)
+            
+            logger.info(f"File processing completed successfully: {result}")
+            
+            return {
+                'statusCode': 200,
+                'body': {
+                    'message': 'SQL file executed successfully',
+                    's3Key': s3_key,
+                    'result': result
+                }
+            }
+        
+        finally:
+            db_connection.close()
+    
+    except Exception as e:
+        logger.error(f"Error processing file: {str(e)}")
+        return {
+            'statusCode': 500,
+            'body': {
+                'error': str(e),
+                's3Key': event.get('s3Key', 'unknown')
+            }
+        }

package/lib/utilities/data-loader-lambda/requirements.txt

@@ -0,0 +1,3 @@
+boto3>=1.26.0
+pymysql>=1.0.2
+psycopg2-binary>=2.9.5