@digitraffic/common 2022.10.25-1 → 2022.10.31-1

package/src/aws/infra/stack/stack-checking-aspect.ts

@@ -0,0 +1,279 @@
+import { Annotations, IAspect, Stack } from "aws-cdk-lib";
+import { CfnFunction, Runtime } from "aws-cdk-lib/aws-lambda";
+import { CfnBucket } from "aws-cdk-lib/aws-s3";
+import { DigitrafficStack, SOLUTION_KEY } from "./stack";
+import { IConstruct } from "constructs";
+import { CfnMethod, CfnResource } from "aws-cdk-lib/aws-apigateway";
+import { paramCase, snakeCase } from "change-case";
+import { CfnQueue } from "aws-cdk-lib/aws-sqs";
+import { LogRetention } from "aws-cdk-lib/aws-logs";
+import IntegrationProperty = CfnMethod.IntegrationProperty;
+
+const MAX_CONCURRENCY_LIMIT = 100;
+const NODE_RUNTIME = Runtime.NODEJS_14_X.name;
+
+enum ResourceType {
+    stackName = "STACK_NAME",
+    reservedConcurrentConcurrency = "RESERVED_CONCURRENT_CONCURRENCY",
+    functionTimeout = "FUNCTION_TIMEOUT",
+    functionMemorySize = "FUNCTION_MEMORY_SIZE",
+    functionRuntime = "FUNCTION_RUNTIME",
+    functionName = "FUNCTION_NAME",
+    tagSolution = "TAG_SOLUTION",
+    bucketPublicity = "BUCKET_PUBLICITY",
+    resourcePath = "RESOURCE_PATH",
+    queueEncryption = "QUEUE_ENCRYPTION",
+    logGroupRetention = "LOG_GROUP_RETENTION",
+}
+
+export class StackCheckingAspect implements IAspect {
+    private readonly stackShortName?: string;
+    private readonly whitelistedResources?: string[];
+
+    constructor(stackShortName?: string, whitelistedResources?: string[]) {
+        this.stackShortName = stackShortName;
+        this.whitelistedResources = whitelistedResources;
+    }
+
+    static create(stack: DigitrafficStack) {
+        return new StackCheckingAspect(
+            stack.configuration.shortName,
+            stack.configuration.whitelistedResources
+        );
+    }
+
+    public visit(node: IConstruct): void {
+        //console.info("visiting class " + node.constructor.name);
+
+        this.checkStack(node);
+        this.checkFunction(node);
+        this.checkTags(node);
+        this.checkBucket(node);
+        this.checkResourceCasing(node);
+        this.checkQueueEncryption(node);
+        this.checkLogGroupRetention(node);
+    }
+
+    private isWhitelisted(key: string) {
+        return this.whitelistedResources?.some((wl) => {
+            return key.matchAll(new RegExp(wl, "g"));
+        });
+    }
+
+    private addAnnotation(
+        node: IConstruct,
+        key: ResourceType | string,
+        message: string,
+        isError = true
+    ) {
+        const resourceKey = `${node.node.path}/${key}`;
+        const isWhiteListed = this.isWhitelisted(resourceKey);
+        const annotationMessage = `${resourceKey}:${message}`;
+
+        // error && whitelisted -> warning
+        // warning && whitelisted -> nothing
+        if (isError && !isWhiteListed) {
+            Annotations.of(node).addError(annotationMessage);
+        } else if ((!isError && !isWhiteListed) || (isError && isWhiteListed)) {
+            Annotations.of(node).addWarning(annotationMessage);
+        }
+    }
+
+    private checkStack(node: IConstruct) {
+        if (node instanceof DigitrafficStack) {
+            if (
+                (node.stackName.includes("Test") ||
+                    node.stackName.includes("Tst")) &&
+                node.configuration.production
+            ) {
+                this.addAnnotation(
+                    node,
+                    ResourceType.stackName,
+                    "Production is set for Test-stack"
+                );
+            }
+
+            if (
+                (node.stackName.includes("Prod") ||
+                    node.stackName.includes("Prd")) &&
+                !node.configuration.production
+            ) {
+                this.addAnnotation(
+                    node,
+                    ResourceType.stackName,
+                    "Production is not set for Production-stack"
+                );
+            }
+        }
+    }
+
+    private checkFunction(node: IConstruct) {
+        if (node instanceof CfnFunction) {
+            if (!node.reservedConcurrentExecutions) {
+                this.addAnnotation(
+                    node,
+                    ResourceType.reservedConcurrentConcurrency,
+                    "Function must have reservedConcurrentConcurrency"
+                );
+            } else if (
+                node.reservedConcurrentExecutions > MAX_CONCURRENCY_LIMIT
+            ) {
+                this.addAnnotation(
+                    node,
+                    ResourceType.reservedConcurrentConcurrency,
+                    "Function reservedConcurrentConcurrency too high!"
+                );
+            }
+
+            if (!node.timeout) {
+                this.addAnnotation(
+                    node,
+                    ResourceType.functionTimeout,
+                    "Function must have timeout"
+                );
+            }
+
+            if (!node.memorySize) {
+                this.addAnnotation(
+                    node,
+                    ResourceType.functionMemorySize,
+                    "Function must have memorySize"
+                );
+            }
+
+            if (node.runtime !== NODE_RUNTIME) {
+                this.addAnnotation(
+                    node,
+                    ResourceType.functionRuntime,
+                    `Function has wrong runtime ${node.runtime}!`
+                );
+            }
+
+            if (
+                this.stackShortName &&
+                node.functionName &&
+                !node.functionName.startsWith(this.stackShortName)
+            ) {
+                this.addAnnotation(
+                    node,
+                    ResourceType.functionName,
+                    `Function name does not begin with ${this.stackShortName}`
+                );
+            }
+        }
+    }
+
+    private checkTags(node: IConstruct) {
+        if (node instanceof Stack) {
+            if (!node.tags.tagValues()[SOLUTION_KEY]) {
+                this.addAnnotation(
+                    node,
+                    ResourceType.tagSolution,
+                    "Solution tag is missing"
+                );
+            }
+        }
+    }
+
+    private checkBucket(node: IConstruct) {
+        if (node instanceof CfnBucket) {
+            const c =
+                node.publicAccessBlockConfiguration as CfnBucket.PublicAccessBlockConfigurationProperty;
+
+            if (c) {
+                if (
+                    !c.blockPublicAcls ||
+                    !c.blockPublicPolicy ||
+                    !c.ignorePublicAcls ||
+                    !c.restrictPublicBuckets
+                ) {
+                    this.addAnnotation(
+                        node,
+                        ResourceType.bucketPublicity,
+                        "Check bucket publicity"
+                    );
+                }
+            }
+        }
+    }
+
+    private static isValidPath(path: string): boolean {
+        // if path includes . or { check only the trailing part of path
+        if (path.includes(".")) {
+            return this.isValidPath(path.split(".")[0]);
+        }
+
+        if (path.includes("{")) {
+            return this.isValidPath(path.split("{")[0]);
+        }
+
+        return paramCase(path) === path;
+    }
+
+    private static isValidQueryString(name: string) {
+        return snakeCase(name) === name;
+    }
+
+    private checkResourceCasing(node: IConstruct) {
+        if (node instanceof CfnResource) {
+            if (!StackCheckingAspect.isValidPath(node.pathPart)) {
+                this.addAnnotation(
+                    node,
+                    ResourceType.resourcePath,
+                    "Path part should be in kebab-case"
+                );
+            }
+        } else if (node instanceof CfnMethod) {
+            const integration = node.integration as IntegrationProperty;
+
+            if (integration && integration.requestParameters) {
+                Object.keys(integration.requestParameters).forEach((key) => {
+                    const split = key.split(".");
+                    const type = split[2];
+                    const name = split[3];
+
+                    if (
+                        type === "querystring" &&
+                        !StackCheckingAspect.isValidQueryString(name)
+                    ) {
+                        this.addAnnotation(
+                            node,
+                            name,
+                            "Querystring should be in snake_case"
+                        );
+                    }
+                });
+            }
+        }
+    }
+
+    private checkQueueEncryption(node: IConstruct) {
+        if (node instanceof CfnQueue) {
+            if (!node.kmsMasterKeyId) {
+                this.addAnnotation(
+                    node,
+                    ResourceType.queueEncryption,
+                    "Queue must have encryption enabled"
+                );
+            }
+        }
+    }
+
+    private checkLogGroupRetention(node: IConstruct) {
+        if (node instanceof LogRetention) {
+            const child = node.node.defaultChild as unknown as Record<
+                string,
+                Record<string, string>
+            >;
+            const retention = child._cfnProperties.RetentionInDays;
+
+            if (!retention) {
+                this.addAnnotation(
+                    node,
+                    ResourceType.logGroupRetention,
+                    "Log group must define log group retention"
+                );
+            }
+        }
+    }
+}

package/src/aws/infra/stack/stack.ts

@@ -0,0 +1,145 @@
+import { Aspects, Stack, StackProps } from "aws-cdk-lib";
+import { IVpc, SecurityGroup, Vpc } from "aws-cdk-lib/aws-ec2";
+import { ISecurityGroup } from "aws-cdk-lib/aws-ec2/lib/security-group";
+import { ITopic, Topic } from "aws-cdk-lib/aws-sns";
+import { StringParameter } from "aws-cdk-lib/aws-ssm";
+import { ISecret, Secret } from "aws-cdk-lib/aws-secretsmanager";
+import { Function as AWSFunction } from "aws-cdk-lib/aws-lambda";
+
+import { StackCheckingAspect } from "./stack-checking-aspect";
+import { Construct } from "constructs";
+import { TrafficType } from "../../../types/traffictype";
+import { DBLambdaEnvironment } from "./lambda-configs";
+
+const SSM_ROOT = "/digitraffic";
+export const SOLUTION_KEY = "Solution";
+const MONITORING_ROOT = "/monitoring";
+
+export const SSM_KEY_WARNING_TOPIC = `${SSM_ROOT}${MONITORING_ROOT}/warning-topic`;
+export const SSM_KEY_ALARM_TOPIC = `${SSM_ROOT}${MONITORING_ROOT}/alarm-topic`;
+
+export interface StackConfiguration {
+    readonly shortName: string;
+    readonly secretId?: string;
+    readonly alarmTopicArn: string;
+    readonly warningTopicArn: string;
+    readonly logsDestinationArn?: string;
+
+    readonly vpcId?: string;
+    readonly lambdaDbSgId?: string;
+    readonly privateSubnetIds?: string[];
+    readonly availabilityZones?: string[];
+
+    readonly trafficType: TrafficType;
+    readonly production: boolean;
+    readonly stackProps: StackProps;
+
+    readonly stackFeatures?: {
+        readonly enableCanaries?: boolean;
+        readonly enableDocumentation?: boolean;
+    };
+
+    /// whitelist resources for StackCheckingAspect
+    readonly whitelistedResources?: string[];
+}
+
+export class DigitrafficStack extends Stack {
+    readonly vpc?: IVpc;
+    readonly lambdaDbSg?: ISecurityGroup;
+    readonly alarmTopic: ITopic;
+    readonly warningTopic: ITopic;
+    readonly secret?: ISecret;
+
+    readonly configuration: StackConfiguration;
+
+    constructor(
+        scope: Construct,
+        id: string,
+        configuration: StackConfiguration
+    ) {
+        super(scope, id, configuration.stackProps);
+
+        this.configuration = configuration;
+
+        if (configuration.secretId) {
+            this.secret = Secret.fromSecretNameV2(
+                this,
+                "Secret",
+                configuration.secretId
+            );
+        }
+
+        // VPC reference construction requires vpcId and availability zones
+        // private subnets are used in Lambda configuration
+        if (configuration.vpcId) {
+            this.vpc = Vpc.fromVpcAttributes(this, "vpc", {
+                vpcId: configuration.vpcId,
+                privateSubnetIds: configuration.privateSubnetIds,
+                availabilityZones: configuration.availabilityZones ?? [],
+            });
+        }
+
+        // security group that allows Lambda database access
+        if (configuration.lambdaDbSgId) {
+            this.lambdaDbSg = SecurityGroup.fromSecurityGroupId(
+                this,
+                "LambdaDbSG",
+                configuration.lambdaDbSgId
+            );
+        }
+
+        this.alarmTopic = Topic.fromTopicArn(
+            this,
+            "AlarmTopic",
+            StringParameter.fromStringParameterName(
+                this,
+                "AlarmTopicParam",
+                SSM_KEY_ALARM_TOPIC
+            ).stringValue
+        );
+        this.warningTopic = Topic.fromTopicArn(
+            this,
+            "WarningTopic",
+            StringParameter.fromStringParameterName(
+                this,
+                "WarningTopicParam",
+                SSM_KEY_WARNING_TOPIC
+            ).stringValue
+        );
+
+        this.addAspects();
+    }
+
+    addAspects() {
+        Aspects.of(this).add(StackCheckingAspect.create(this));
+    }
+
+    createLambdaEnvironment(): DBLambdaEnvironment {
+        return this.createDefaultLambdaEnvironment(
+            this.configuration.shortName
+        );
+    }
+
+    createDefaultLambdaEnvironment(dbApplication: string): DBLambdaEnvironment {
+        return this.configuration.secretId
+            ? {
+                  SECRET_ID: this.configuration.secretId,
+                  DB_APPLICATION: dbApplication,
+              }
+            : {
+                  DB_APPLICATION: dbApplication,
+              };
+    }
+
+    getSecret(): ISecret {
+        if (this.secret === undefined) {
+            throw new Error("Secret is undefined");
+        }
+        return this.secret;
+    }
+
+    grantSecret(...lambdas: AWSFunction[]) {
+        const secret = this.getSecret();
+        lambdas.forEach((l: AWSFunction) => secret.grantRead(l));
+    }
+}

package/src/aws/infra/stack/subscription.ts

@@ -0,0 +1,58 @@
+import { CfnSubscriptionFilter } from "aws-cdk-lib/aws-logs";
+import { Function as AWSFunction } from "aws-cdk-lib/aws-lambda";
+import { DigitrafficStack } from "./stack";
+import { Construct } from "constructs";
+import { MonitoredFunction } from "./monitoredfunction";
+
+/**
+ * Creates a subscription filter that subscribes to a Lambda Log Group and delivers the logs to another destination.
+ * https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-logs-subscriptionfilter.html
+ * @param lambda The Lambda function, needed to create a dependency
+ * @param lambdaName The Lambda name from which the Log Group name is derived
+ * @param logDestinationArn Destination for streamed logs
+ * @param stack CloudFormation stack
+ */
+export function createSubscription(
+    lambda: AWSFunction,
+    lambdaName: string,
+    logDestinationArn: string | undefined,
+    stack: Construct
+): CfnSubscriptionFilter | undefined {
+    if (logDestinationArn == undefined) {
+        return undefined;
+    }
+    const filter = new CfnSubscriptionFilter(
+        stack,
+        `${lambdaName}LogsSubscription`,
+        {
+            logGroupName: `/aws/lambda/${lambdaName}`,
+            filterPattern: "",
+            destinationArn: logDestinationArn,
+        }
+    );
+
+    filter.node.addDependency(lambda);
+
+    return filter;
+}
+
+export class DigitrafficLogSubscriptions {
+    constructor(stack: DigitrafficStack, ...lambdas: MonitoredFunction[]) {
+        const destinationArn = stack.configuration.logsDestinationArn;
+        if (destinationArn !== undefined) {
+            lambdas.forEach((lambda) => {
+                const filter = new CfnSubscriptionFilter(
+                    stack,
+                    `${lambda.givenName}LogsSubscription`,
+                    {
+                        logGroupName: `/aws/lambda/${lambda.givenName}`,
+                        filterPattern: "",
+                        destinationArn,
+                    }
+                );
+
+                filter.node.addDependency(lambda);
+            });
+        }
+    }
+}

package/src/aws/infra/usage-plans.ts

@@ -0,0 +1,41 @@
+import {IApiKey, RestApi} from 'aws-cdk-lib/aws-apigateway';
+
+/**
+ * Creates an usage plan for a REST API with a single API key
+ * @param api The REST API
+ * @param apiKeyId Id for the API key, this is a surrogate id for CDK, not displayed anywhere
+ * @param apiKeyName Name for the API key, this is displayed in the AWS Console
+ * @deprecated Creates randomized API key names, use createDefaultUsagePlan instead
+ */
+export function createUsagePlan(api: RestApi, apiKeyId: string, apiKeyName: string): IApiKey {
+    const apiKey = api.addApiKey(apiKeyId);
+    const plan = api.addUsagePlan(apiKeyName, {
+        name: apiKeyName,
+    });
+    plan.addApiStage({
+        stage: api.deploymentStage,
+    });
+    plan.addApiKey(apiKey);
+
+    return apiKey;
+}
+
+/**
+ * Creates a default usage plan for a REST API with a single API key
+ * @param api The REST API
+ * @param apiName Name of the api. Will generate key: apiName + ' API Key' and plan: apiName + ' API Usage Plan'
+ */
+export function createDefaultUsagePlan(api: RestApi, apiName: string): IApiKey {
+    const apiKeyName = apiName + ' API Key';
+    const usagePlanName = apiName + ' API Usage Plan';
+    const apiKey = api.addApiKey(apiKeyName, { apiKeyName: apiKeyName });
+    const plan = api.addUsagePlan(usagePlanName, {
+        name: usagePlanName,
+    });
+    plan.addApiStage({
+        stage: api.deploymentStage,
+    });
+    plan.addApiKey(apiKey);
+
+    return apiKey;
+}

package/src/aws/runtime/apikey.ts

@@ -0,0 +1,9 @@
+import {APIGateway} from "aws-sdk";
+
+export function getApiKeyFromAPIGateway(keyId: string): Promise<APIGateway.Types.ApiKey> {
+    const agw = new APIGateway();
+    return agw.getApiKey({
+        apiKey: keyId,
+        includeValue: true,
+    }).promise();
+}

package/src/aws/runtime/digitraffic-integration-response.ts

@@ -0,0 +1,28 @@
+import {IntegrationResponse} from "aws-cdk-lib/aws-apigateway";
+import {MediaType} from "../types/mediatypes";
+import {RESPONSE_DEFAULT_LAMBDA} from "../infra/api/response";
+
+export abstract class DigitrafficIntegrationResponse {
+
+    static ok(mediaType: MediaType): IntegrationResponse {
+        return this.create("200", mediaType);
+    }
+
+    static badRequest(mediaType?: MediaType): IntegrationResponse {
+        return this.create("400", mediaType ?? MediaType.TEXT_PLAIN);
+    }
+
+    static notImplemented(mediaType?: MediaType): IntegrationResponse {
+        return this.create("501", mediaType ?? MediaType.TEXT_PLAIN);
+    }
+
+    static create(statusCode: string, mediaType: MediaType): IntegrationResponse {
+        return {
+            statusCode,
+            responseTemplates: {
+                [mediaType]: RESPONSE_DEFAULT_LAMBDA,
+            },
+        };
+    }
+}
+

package/src/aws/runtime/environment.ts

@@ -0,0 +1,9 @@
+export function envValue(key: string): string {
+    const value = process.env[key];
+
+    if (value == null) {
+        throw new Error(`Missing environment value ${key}`);
+    }
+
+    return value;
+}

package/src/aws/runtime/messaging.ts

@@ -0,0 +1,26 @@
+import {SNS} from "aws-sdk";
+
+/**
+ * Utility function for publishing SNS messages.
+ * Made because using *await* with AWS APIs doesn't require calling promise() but nothing works if it isn't called.
+ * Retries a single time in case of failure.
+ * @param message
+ * @param topicArn
+ * @param sns
+ */
+export async function snsPublish(message: string, topicArn: string, sns: SNS) {
+    const publishParams = {
+        Message: message,
+        TopicArn: topicArn,
+    };
+    try {
+        await sns.publish(publishParams).promise();
+    } catch (error) {
+        console.error('method=snsPublish error, retrying', error);
+        try {
+            await sns.publish(publishParams).promise();
+        } catch (e2) {
+            console.error('method=snsPublish error after retry', e2);
+        }
+    }
+}

package/src/aws/runtime/s3.ts

@@ -0,0 +1,44 @@
+import {S3} from "aws-sdk";
+
+export async function uploadToS3<Body extends S3.Body | undefined>(
+    bucketName: string,
+    body: Body,
+    objectName: string,
+    cannedAcl?: string,
+    contentType?: string,
+) {
+
+    const s3 = new S3();
+    try {
+        await doUpload(
+            s3, bucketName, body, objectName, cannedAcl, contentType,
+        );
+    } catch (error) {
+        console.warn('method=uploadToS3 retrying upload to bucket %s', bucketName);
+        try {
+            await doUpload(
+                s3, bucketName, body, objectName, cannedAcl, contentType,
+            );
+        } catch (e2) {
+            console.error('method=uploadToS3 failed retrying upload to bucket %s', bucketName);
+        }
+    }
+}
+
+function doUpload<Body extends S3.Body | undefined>(
+    s3: S3,
+    bucketName: string,
+    body: Body,
+    filename: string,
+    cannedAcl?: string,
+    contentType?: string,
+) {
+
+    return s3.upload({
+        Bucket: bucketName,
+        Body: body,
+        Key: filename,
+        ACL: cannedAcl,
+        ContentType: contentType,
+    }).promise();
+}