@digitraffic/common 2022.10.25-1 → 2022.10.31-1

package/src/aws/infra/sqs-integration.ts

@@ -0,0 +1,102 @@
+import {Aws} from "aws-cdk-lib";
+import {AwsIntegration, PassthroughBehavior, RequestValidator, Resource} from "aws-cdk-lib/aws-apigateway";
+import {Queue} from "aws-cdk-lib/aws-sqs";
+import {PolicyStatement, Role, ServicePrincipal} from "aws-cdk-lib/aws-iam";
+import {IModel} from "aws-cdk-lib/aws-apigateway/lib/model";
+import {Construct} from "constructs";
+
+export function attachQueueToApiGatewayResource(
+    stack: Construct,
+    queue: Queue,
+    resource: Resource,
+    requestValidator: RequestValidator,
+    resourceName: string,
+    apiKeyRequired: boolean,
+    requestModels?: {[param: string]: IModel},
+) {
+    // role for API Gateway
+    const apiGwRole = new Role(stack, `${resourceName}APIGatewayToSQSRole`, {
+        assumedBy: new ServicePrincipal('apigateway.amazonaws.com'),
+    });
+    // grants API Gateway the right to send SQS messages
+    apiGwRole.addToPolicy(new PolicyStatement({
+        resources: [
+            queue.queueArn,
+        ],
+        actions: [
+            'sqs:SendMessage',
+        ],
+    }));
+    // grants API Gateway the right write CloudWatch Logs
+    apiGwRole.addToPolicy(new PolicyStatement({
+        resources: [
+            '*',
+        ],
+        actions: [
+            'logs:CreateLogGroup',
+            'logs:CreateLogStream',
+            'logs:DescribeLogGroups',
+            'logs:DescribeLogStreams',
+            'logs:PutLogEvents',
+            'logs:GetLogEvents',
+            'logs:FilterLogEvents',
+        ],
+    }));
+    // create an integration between API Gateway and an SQS queue
+    const fifoMessageGroupId = queue.fifo ? '&MessageGroupId=AlwaysSameFifoGroup' : '';
+    const sqsIntegration = new AwsIntegration({
+        service: 'sqs',
+        integrationHttpMethod: 'POST',
+        options: {
+            passthroughBehavior: PassthroughBehavior.NEVER,
+            credentialsRole: apiGwRole,
+            requestParameters: {
+                // SQS requires the Content-Type of the HTTP request to be application/x-www-form-urlencoded
+                'integration.request.header.Content-Type': "'application/x-www-form-urlencoded'",
+            },
+            requestTemplates: {
+                // map the JSON request to a form parameter, FIFO needs also MessageGroupId
+                // https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html
+                'application/json': `Action=SendMessage${fifoMessageGroupId}&MessageBody=$util.urlEncode($input.body)`,
+            },
+            // these are required by SQS
+            integrationResponses: [
+                {
+                    statusCode: '200',
+                    responseTemplates: {
+                        'text/html': 'Success',
+                    },
+                },
+                {
+                    statusCode: '500',
+                    responseTemplates: {
+                        'text/html': 'Error',
+                    },
+                    selectionPattern: '500',
+                },
+
+            ],
+
+        },
+        path: `${Aws.ACCOUNT_ID}/${queue.queueName}`,
+    });
+    resource.addMethod('POST', sqsIntegration, {
+        requestValidator,
+        apiKeyRequired,
+        requestModels: requestModels ?? {},
+        methodResponses: [
+            {
+                statusCode: '200',
+                responseParameters: {
+                    'method.response.header.Content-Type': true,
+                },
+            },
+            {
+                statusCode: '500',
+                responseParameters: {
+                    'method.response.header.Content-Type': true,
+                },
+            },
+        ],
+    });
+}

package/src/aws/infra/sqs-queue.ts

@@ -0,0 +1,148 @@
+import {Queue, QueueEncryption, QueueProps} from "aws-cdk-lib/aws-sqs";
+import {Duration} from "aws-cdk-lib";
+import {BlockPublicAccess, Bucket} from "aws-cdk-lib/aws-s3";
+import {PolicyStatement} from "aws-cdk-lib/aws-iam";
+import {InlineCode, Runtime} from "aws-cdk-lib/aws-lambda";
+import {RetentionDays} from "aws-cdk-lib/aws-logs";
+import {SqsEventSource} from "aws-cdk-lib/aws-lambda-event-sources";
+import {ComparisonOperator, TreatMissingData} from "aws-cdk-lib/aws-cloudwatch";
+import {SnsAction} from "aws-cdk-lib/aws-cloudwatch-actions";
+import {ManagedUpload} from "aws-sdk/clients/s3";
+import {S3} from "aws-sdk";
+import {SQSEvent, SQSHandler, SQSRecord} from "aws-lambda";
+import {Construct} from "constructs";
+import {DigitrafficStack} from "./stack/stack";
+import {MonitoredFunction} from "./stack/monitoredfunction";
+
+/**
+ * Construct for creating SQS-queues.
+ *
+ * If you don't config your own deadLetterQueue, this will create a dlq for you, also a lambda function, a s3 bucket
+ * and an alarm for the queue.  Anything that goes to the dlq will be written into the bucket and the alarm is activated.
+ */
+export class DigitrafficSqsQueue extends Queue {
+    constructor(scope: Construct, name: string, props: QueueProps) {
+        super(scope, name, props);
+    }
+
+    static create(stack: DigitrafficStack, name: string, props: QueueProps): DigitrafficSqsQueue {
+        const queueName = `${stack.configuration.shortName}-${name}-Queue`;
+        const queueProps = {...props, ...{
+            encryption: QueueEncryption.KMS_MANAGED,
+            queueName,
+            deadLetterQueue: props.deadLetterQueue || {
+                maxReceiveCount: 2,
+                queue: DigitrafficDLQueue.create(stack, name),
+            },
+        }};
+
+        return new DigitrafficSqsQueue(stack, queueName, queueProps);
+    }
+}
+
+export class DigitrafficDLQueue {
+    static create(stack: DigitrafficStack, name: string): DigitrafficSqsQueue {
+        const dlqName = `${stack.configuration.shortName}-${name}-DLQ`;
+
+        const dlq = new DigitrafficSqsQueue(stack, dlqName, {
+            queueName: dlqName,
+            visibilityTimeout: Duration.seconds(60),
+            encryption: QueueEncryption.KMS_MANAGED,
+        });
+
+        const dlqBucket = new Bucket(stack, `${dlqName}-Bucket`, {
+            blockPublicAccess: BlockPublicAccess.BLOCK_ALL,
+        });
+
+        const dlqFunctionName = `${dlqName}-Function`;
+        const lambda = MonitoredFunction.create(stack, dlqFunctionName, {
+            runtime: Runtime.NODEJS_14_X,
+            logRetention: RetentionDays.ONE_YEAR,
+            functionName: dlqFunctionName,
+            code: getDlqCode(dlqBucket.bucketName),
+            timeout: Duration.seconds(10),
+            handler: 'index.handler',
+            memorySize: 128,
+            reservedConcurrentExecutions: 1,
+        });
+
+        const statement = new PolicyStatement();
+        statement.addActions('s3:PutObject');
+        statement.addActions('s3:PutObjectAcl');
+        statement.addResources(dlqBucket.bucketArn + '/*');
+
+        lambda.addToRolePolicy(statement);
+        lambda.addEventSource(new SqsEventSource(dlq));
+
+        addDLQAlarm(stack, dlqName, dlq);
+
+        return dlq;
+    }
+}
+
+function addDLQAlarm(stack: DigitrafficStack, dlqName: string, dlq: Queue) {
+    const alarmName = `${dlqName}-Alarm`;
+    dlq.metricNumberOfMessagesReceived({
+        period: Duration.minutes(5),
+    }).createAlarm(stack, alarmName, {
+        alarmName,
+        threshold: 0,
+        evaluationPeriods: 1,
+        treatMissingData: TreatMissingData.NOT_BREACHING,
+        comparisonOperator: ComparisonOperator.GREATER_THAN_THRESHOLD,
+    }).addAlarmAction(new SnsAction(stack.warningTopic));
+}
+
+function getDlqCode(bName: string): InlineCode {
+    const functionBody = DLQ_LAMBDA_CODE
+        .replace("__bucketName__", bName)
+        .replace("__upload__", uploadToS3.toString())
+        .replace("__doUpload__", doUpload.toString())
+        .replace("__handler__", createHandler().toString().substring(23)); // remove function handler() from signature
+
+    return new InlineCode(functionBody);
+}
+
+async function uploadToS3(s3: S3, bName: string, body: string, objectName: string): Promise<void> {
+    try {
+        console.info('writing %s to %s', objectName, bName);
+        await doUpload(s3, bName, body, objectName);
+    } catch (error) {
+        console.warn(error);
+        console.warn('method=uploadToS3 retrying upload to bucket %s', bName);
+        try {
+            await doUpload(s3, bName, body, objectName);
+        } catch (e2) {
+            console.error('method=uploadToS3 failed retrying upload to bucket %s', bName);
+        }
+    }
+}
+
+function doUpload(s3: S3, bName: string, Body: string, Key: string): Promise<ManagedUpload.SendData> {
+    return s3.upload({
+        Bucket: bName, Body, Key,
+    }).promise();
+}
+
+// bucketName is unused, will be overridden in the actual lambda code below
+const bucketName = '';
+
+function createHandler(): SQSHandler {
+    return async function handler(event: SQSEvent): Promise<void> {
+        // eslint-disable-next-line @typescript-eslint/no-var-requires
+        const AWS = require('aws-sdk');
+
+        const millis = new Date().getTime();
+        await Promise.all(event.Records.map((e: SQSRecord, idx: number) =>
+            uploadToS3(new AWS.S3(), bucketName, e.body, `dlq-${millis}-${idx}.json`)));
+    };
+}
+
+const DLQ_LAMBDA_CODE = `const AWS = require('aws-sdk');
+const bucketName = "__bucketName__";
+
+__upload__
+__doUpload__
+
+exports.handler = async (event) => __handler__
+`;

package/src/aws/infra/stack/lambda-configs.ts

@@ -0,0 +1,207 @@
+import {
+    Architecture,
+    AssetCode,
+    Code,
+    FunctionProps,
+    Runtime,
+} from "aws-cdk-lib/aws-lambda";
+import { Duration } from "aws-cdk-lib";
+import { ISecurityGroup, IVpc, SubnetSelection } from "aws-cdk-lib/aws-ec2";
+import { RetentionDays } from "aws-cdk-lib/aws-logs";
+import { Role } from "aws-cdk-lib/aws-iam";
+import { DigitrafficStack } from "./stack";
+import { MonitoredFunctionAlarmProps } from "./monitoredfunction";
+
+export type LambdaEnvironment = Record<string, string>;
+
+export type DBLambdaEnvironment = LambdaEnvironment & {
+    SECRET_ID?: string;
+    DB_APPLICATION: string;
+};
+
+export interface LambdaConfiguration {
+    vpcId: string;
+    allowFromIpAddresses?: string[];
+    privateSubnetIds: string[];
+    availabilityZones: string[];
+    lambdaDbSgId: string;
+    dbProps?: DbProps;
+    defaultLambdaDurationSeconds?: number;
+    logsDestinationArn: string;
+    memorySize?: number;
+    runtime?: Runtime;
+}
+
+declare interface DbProps {
+    username: string;
+    password: string;
+    uri?: string;
+    ro_uri?: string;
+}
+
+export function databaseFunctionProps(
+    stack: DigitrafficStack,
+    environment: LambdaEnvironment,
+    lambdaName: string,
+    simpleLambdaName: string,
+    config?: Partial<FunctionParameters>
+): FunctionProps {
+    const vpcSubnets = stack.vpc
+        ? {
+              subnets: stack.vpc.privateSubnets,
+          }
+        : undefined;
+
+    return {
+        ...lambdaFunctionProps(
+            stack,
+            environment,
+            lambdaName,
+            simpleLambdaName,
+            config
+        ),
+        ...{
+            vpc: stack.vpc || undefined,
+            vpcSubnets,
+            securityGroup: stack.lambdaDbSg || undefined,
+        },
+    };
+}
+
+export function lambdaFunctionProps(
+    stack: DigitrafficStack,
+    environment: LambdaEnvironment,
+    lambdaName: string,
+    simpleLambdaName: string,
+    config?: Partial<FunctionParameters>
+): FunctionProps {
+    return {
+        runtime: config?.runtime ?? Runtime.NODEJS_14_X,
+        architecture: config?.architecture ?? Architecture.ARM_64,
+        memorySize: config?.memorySize ?? 128,
+        functionName: lambdaName,
+        role: config?.role,
+        timeout: Duration.seconds(config?.timeout ?? 60),
+        logRetention: RetentionDays.ONE_YEAR,
+        reservedConcurrentExecutions: config?.reservedConcurrentExecutions ?? 2,
+        code: getAssetCode(simpleLambdaName, config?.singleLambda ?? false),
+        handler: `${simpleLambdaName}.handler`,
+        environment,
+    };
+}
+
+function getAssetCode(
+    simpleLambdaName: string,
+    isSingleLambda: boolean
+): AssetCode {
+    const lambdaPath = isSingleLambda
+        ? `dist/lambda/`
+        : `dist/lambda/${simpleLambdaName}`;
+
+    return new AssetCode(lambdaPath);
+}
+
+/**
+ * Creates a base configuration for a Lambda that uses an RDS database
+ * @param vpc "Private" Lambdas are associated with a VPC
+ * @param lambdaDbSg Security Group shared by Lambda and RDS
+ * @param props Database connection properties for the Lambda
+ * @param config Lambda configuration
+ */
+export function dbLambdaConfiguration(
+    vpc: IVpc,
+    lambdaDbSg: ISecurityGroup,
+    props: LambdaConfiguration,
+    config: FunctionParameters
+): FunctionProps {
+    return {
+        runtime: props.runtime || Runtime.NODEJS_14_X,
+        memorySize: props.memorySize || config.memorySize || 1024,
+        functionName: config.functionName,
+        code: config.code,
+        role: config.role,
+        handler: config.handler,
+        timeout: Duration.seconds(
+            config.timeout || props.defaultLambdaDurationSeconds || 60
+        ),
+        environment: config.environment || {
+            DB_USER: props.dbProps?.username ?? "",
+            DB_PASS: props.dbProps?.password ?? "",
+            DB_URI:
+                (config.readOnly
+                    ? props.dbProps?.ro_uri
+                    : props.dbProps?.uri) ?? "",
+        },
+        logRetention: RetentionDays.ONE_YEAR,
+        vpc: vpc,
+        vpcSubnets: {
+            subnets: vpc.privateSubnets,
+        },
+        securityGroups: [lambdaDbSg],
+        reservedConcurrentExecutions: config.reservedConcurrentExecutions || 3,
+    };
+}
+
+export function defaultLambdaConfiguration(
+    config: FunctionParameters
+): FunctionProps {
+    const props: FunctionProps = {
+        runtime: Runtime.NODEJS_14_X,
+        memorySize: config.memorySize ?? 128,
+        functionName: config.functionName,
+        handler: config.handler,
+        environment: config.environment ?? {},
+        logRetention: RetentionDays.ONE_YEAR,
+        reservedConcurrentExecutions: config.reservedConcurrentExecutions,
+        code: config.code,
+        role: config.role,
+        timeout: Duration.seconds(config.timeout || 10),
+    };
+    if (config.vpc) {
+        return {
+            ...props,
+            ...{
+                vpc: config.vpc,
+                vpcSubnets: {
+                    subnets: config.vpc?.privateSubnets,
+                },
+            },
+        };
+    }
+    return props;
+}
+
+export interface FunctionParameters {
+    memorySize?: number;
+    timeout?: number;
+    functionName?: string;
+    code: Code;
+    handler: string;
+    readOnly?: boolean;
+    environment?: {
+        [key: string]: string;
+    };
+    reservedConcurrentExecutions?: number;
+    role?: Role;
+    vpc?: IVpc;
+    vpcSubnets?: SubnetSelection;
+    runtime?: Runtime;
+    architecture?: Architecture;
+    singleLambda?: boolean;
+}
+
+export type MonitoredFunctionParameters = FunctionParameters & {
+    readonly memorySize?: number;
+    readonly timeout?: number;
+    readonly functionName?: string;
+    readonly reservedConcurrentExecutions?: number;
+    readonly role?: Role;
+    readonly runtime?: Runtime;
+    readonly architecture?: Architecture;
+    readonly singleLambda?: boolean;
+
+    readonly durationAlarmProps?: MonitoredFunctionAlarmProps;
+    readonly durationWarningProps?: MonitoredFunctionAlarmProps;
+    readonly errorAlarmProps?: MonitoredFunctionAlarmProps;
+    readonly throttleAlarmProps?: MonitoredFunctionAlarmProps;
+};