@digitraffic/common 2022.10.25-1 → 2022.10.28-2

package/src/aws/infra/canaries/database-checker.ts

@@ -0,0 +1,155 @@
+import { DTDatabase, inDatabaseReadonly } from "../../../database/database";
+import { ProxyHolder } from "../../runtime/secrets/proxy-holder";
+import { RdsHolder } from "../../runtime/secrets/rds-holder";
+import { getEnvVariable } from "../../../utils/utils";
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const synthetics = require("Synthetics");
+
+abstract class DatabaseCheck<T> {
+    readonly name: string;
+    readonly sql: string;
+    failed: boolean;
+
+    protected constructor(name: string, sql: string) {
+        this.name = name;
+        this.sql = sql;
+        this.failed = false;
+    }
+
+    abstract check(value: T): void;
+}
+
+// For backwards compatibility disable following rule for BaseResponse.
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+interface BaseResponse {}
+
+interface CountResponse extends BaseResponse {
+    count: number;
+}
+
+class CountDatabaseCheck extends DatabaseCheck<CountResponse> {
+    readonly minCount: number | null;
+    readonly maxCount: number | null;
+
+    constructor(
+        name: string,
+        sql: string,
+        minCount: number | null,
+        maxCount: number | null
+    ) {
+        super(name, sql);
+
+        if (minCount == null && maxCount == null) {
+            throw new Error("no max or min given!");
+        }
+
+        this.minCount = minCount;
+        this.maxCount = maxCount;
+    }
+
+    check(value: CountResponse) {
+        if (!value) {
+            this.failed = true;
+            throw new Error("no return value");
+        } else {
+            if ("count" in value) {
+                if (this.minCount && value.count < this.minCount) {
+                    this.failed = true;
+                    throw new Error(
+                        `count was ${value.count}, minimum is ${this.minCount}`
+                    );
+                }
+                if (this.maxCount && value.count > this.maxCount) {
+                    this.failed = true;
+                    throw new Error(
+                        `count was ${value.count}, max is ${this.maxCount}`
+                    );
+                }
+            } else {
+                this.failed = true;
+
+                console.info("received " + JSON.stringify(value));
+
+                throw new Error("no count available");
+            }
+        }
+    }
+}
+
+const stepConfig = {
+    continueOnStepFailure: true,
+    screenshotOnStepStart: false,
+    screenshotOnStepSuccess: false,
+    screenshotOnStepFailure: false,
+};
+
+export class DatabaseChecker {
+    credentialsFunction: () => Promise<void>;
+    checks: DatabaseCheck<BaseResponse>[];
+
+    private constructor(credentialsFunction: () => Promise<void>) {
+        this.credentialsFunction = credentialsFunction;
+        this.checks = [];
+
+        synthetics.getConfiguration().disableRequestMetrics();
+
+        synthetics.getConfiguration().withFailedCanaryMetric(true);
+    }
+
+    static createForProxy() {
+        return new DatabaseChecker(() =>
+            new ProxyHolder(getEnvVariable("SECRET_ID")).setCredentials()
+        );
+    }
+
+    static createForRds() {
+        return new DatabaseChecker(() =>
+            new RdsHolder(getEnvVariable("SECRET_ID")).setCredentials()
+        );
+    }
+
+    one(name: string, sql: string) {
+        this.checks.push(new CountDatabaseCheck(name, sql, 1, 1));
+
+        return this;
+    }
+
+    empty(name: string, sql: string) {
+        this.checks.push(new CountDatabaseCheck(name, sql, null, 0));
+
+        return this;
+    }
+
+    notEmpty(name: string, sql: string) {
+        this.checks.push(new CountDatabaseCheck(name, sql, 1, null));
+
+        return this;
+    }
+
+    async expect() {
+        if (!this.checks.length) {
+            throw new Error("No checks");
+        }
+
+        await this.credentialsFunction();
+        await inDatabaseReadonly(async (db: DTDatabase) => {
+            for (const check of this.checks) {
+                console.info("canary checking sql " + check.sql);
+
+                const value = await db.oneOrNone(check.sql);
+                const checkFunction = () => {
+                    check.check(value);
+                };
+
+                synthetics.executeStep(check.name, checkFunction, stepConfig);
+            }
+        });
+
+        if (this.checks.some((check) => check.failed)) {
+            throw new Error("Failed");
+        }
+
+        return "OK";
+    }
+}

package/src/aws/infra/canaries/url-canary.ts

@@ -0,0 +1,74 @@
+import { Construct } from "constructs";
+import { CanaryParameters } from "./canary-parameters";
+import { Role } from "aws-cdk-lib/aws-iam";
+import { DigitrafficCanary } from "./canary";
+import { ISecret } from "aws-cdk-lib/aws-secretsmanager";
+import { DigitrafficStack } from "../stack/stack";
+import { LambdaEnvironment } from "../stack/lambda-configs";
+import { DigitrafficRestApi } from "../stack/rest_apis";
+import { ENV_API_KEY, ENV_HOSTNAME, ENV_SECRET } from "./canary-keys";
+
+export interface UrlCanaryParameters extends CanaryParameters {
+    readonly hostname: string;
+    readonly apiKeyId?: string;
+}
+
+export class UrlCanary extends DigitrafficCanary {
+    constructor(
+        stack: Construct,
+        role: Role,
+        params: UrlCanaryParameters,
+        secret?: ISecret
+    ) {
+        const canaryName = `${params.name}-url`;
+        const environmentVariables: LambdaEnvironment = {};
+        environmentVariables[ENV_HOSTNAME] = params.hostname;
+
+        if (params.secret) {
+            environmentVariables[ENV_SECRET] = params.secret;
+        }
+
+        if (params.apiKeyId) {
+            environmentVariables[ENV_API_KEY] = params.apiKeyId;
+        }
+
+        if (secret) {
+            secret.grantRead(role);
+        }
+
+        // the handler code is defined at the actual project using this
+        super(stack, canaryName, role, params, environmentVariables);
+    }
+
+    static create(
+        stack: DigitrafficStack,
+        role: Role,
+        publicApi: DigitrafficRestApi,
+        params: Partial<UrlCanaryParameters>
+    ): UrlCanary {
+        return new UrlCanary(stack, role, {
+            ...{
+                handler: `${params.name}.handler`,
+                hostname: publicApi.hostname(),
+                apiKeyId: this.getApiKey(publicApi),
+            },
+            ...params,
+        } as UrlCanaryParameters);
+    }
+
+    static getApiKey(publicApi: DigitrafficRestApi): string | undefined {
+        const apiKeys = publicApi.apiKeyIds;
+
+        if (apiKeys.length > 1) {
+            console.info("rest api has more than one api key");
+        }
+
+        if (apiKeys.length === 0) {
+            console.info("rest api has no api keys");
+            return undefined;
+        }
+
+        // always use first api key
+        return publicApi.apiKeyIds[0];
+    }
+}

package/src/aws/infra/canaries/url-checker.ts

@@ -0,0 +1,366 @@
+import { IncomingMessage, RequestOptions } from "http";
+import { Asserter } from "../../../test/asserter";
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const synthetics = require("Synthetics");
+import zlib = require("zlib");
+import { MediaType } from "../../types/mediatypes";
+import { getApiKeyFromAPIGateway } from "../../runtime/apikey";
+import { FeatureCollection } from "geojson";
+import { isValidGeoJson } from "../../../utils/geometry";
+import { getEnvVariable } from "../../../utils/utils";
+import { ENV_API_KEY, ENV_HOSTNAME } from "./canary-keys";
+
+export const API_KEY_HEADER = "x-api-key";
+
+const baseHeaders = {
+    "Digitraffic-User": "internal-digitraffic-canary",
+    "Accept-Encoding": "gzip",
+    Accept: "*/*",
+} as Record<string, string>;
+
+type CheckerFunction = (Res: IncomingMessage) => void;
+type JsonCheckerFunction<T> = (
+    json: T,
+    body: string,
+    message: IncomingMessage
+) => void;
+
+export class UrlChecker {
+    private readonly requestOptions: RequestOptions;
+
+    constructor(hostname: string, apiKey?: string) {
+        const headers = { ...baseHeaders };
+
+        if (apiKey) {
+            headers[API_KEY_HEADER] = apiKey;
+        }
+
+        this.requestOptions = {
+            hostname,
+            method: "GET",
+            protocol: "https:",
+            headers,
+        };
+
+        synthetics.getConfiguration().disableRequestMetrics();
+
+        synthetics
+            .getConfiguration()
+            .withIncludeRequestBody(false)
+            .withIncludeRequestHeaders(false)
+            .withIncludeResponseBody(false)
+            .withIncludeResponseHeaders(false)
+            .withFailedCanaryMetric(true);
+    }
+
+    static create(hostname: string, apiKeyId: string): Promise<UrlChecker> {
+        return getApiKeyFromAPIGateway(apiKeyId).then((apiKey) => {
+            return new UrlChecker(hostname, apiKey.value);
+        });
+    }
+
+    static createV2(): Promise<UrlChecker> {
+        return this.create(
+            getEnvVariable(ENV_HOSTNAME),
+            getEnvVariable(ENV_API_KEY)
+        );
+    }
+
+    expectStatus<T>(
+        statusCode: number,
+        url: string,
+        callback: JsonCheckerFunction<T>
+    ): Promise<void> {
+        const requestOptions = {
+            ...this.requestOptions,
+            ...{
+                path: url,
+            },
+        };
+
+        return synthetics.executeHttpStep(
+            `Verify ${statusCode} for ${url.replace(/auth=.*/, "")}`,
+            requestOptions,
+            callback
+        );
+    }
+
+    expect200<T>(
+        url: string,
+        ...callbacks: JsonCheckerFunction<T>[]
+    ): Promise<void> {
+        const callback = async (
+            json: T,
+            body: string,
+            res: IncomingMessage
+        ) => {
+            await Promise.allSettled(callbacks.map((c) => c(json, body, res)));
+        };
+
+        return this.expectStatus(200, url, callback);
+    }
+
+    expect404(url: string): Promise<void> {
+        const requestOptions = {
+            ...this.requestOptions,
+            ...{
+                path: url,
+            },
+        };
+
+        return synthetics.executeHttpStep(
+            `Verify 404 for ${url}`,
+            requestOptions,
+            validateStatusCodeAndContentType(404, MediaType.TEXT_PLAIN)
+        );
+    }
+
+    expect400(url: string): Promise<void> {
+        const requestOptions = {
+            ...this.requestOptions,
+            ...{
+                path: url,
+            },
+        };
+
+        return synthetics.executeHttpStep(
+            `Verify 400 for ${url}`,
+            requestOptions,
+            validateStatusCodeAndContentType(400, MediaType.TEXT_PLAIN)
+        );
+    }
+
+    expect403WithoutApiKey(url: string, mediaType?: MediaType): Promise<void> {
+        if (
+            !this.requestOptions.headers ||
+            !this.requestOptions.headers[API_KEY_HEADER]
+        ) {
+            console.error("No api key defined");
+        }
+
+        const requestOptions = {
+            ...this.requestOptions,
+            ...{
+                path: url,
+                headers: baseHeaders,
+            },
+        };
+
+        return synthetics.executeHttpStep(
+            `Verify 403 for ${url}`,
+            requestOptions,
+            validateStatusCodeAndContentType(
+                403,
+                mediaType || MediaType.APPLICATION_JSON
+            )
+        );
+    }
+
+    done(): string {
+        return "Canary successful";
+    }
+}
+
+async function getResponseBody(response: IncomingMessage): Promise<string> {
+    const body = await getBodyFromResponse(response);
+
+    if (response.headers["content-encoding"] === "gzip") {
+        try {
+            return zlib.gunzipSync(body).toString();
+        } catch (e) {
+            console.info("error " + JSON.stringify(e));
+        }
+    }
+
+    return body.toString();
+}
+
+function getBodyFromResponse(response: IncomingMessage): Promise<string> {
+    return new Promise((resolve: (value: string) => void) => {
+        const buffers: Buffer[] = [];
+
+        response.on("data", (data: Buffer) => {
+            buffers.push(data);
+        });
+
+        response.on("end", () => {
+            resolve(Buffer.concat(buffers).toString());
+        });
+    });
+}
+
+/**
+ * Returns function, that validates that the status code and content-type from response are the given values
+ * @param statusCode
+ * @param contentType
+ */
+function validateStatusCodeAndContentType(
+    statusCode: number,
+    contentType: MediaType
+): (Res: IncomingMessage) => Promise<void> {
+    return (res: IncomingMessage) => {
+        return new Promise((resolve) => {
+            if (res.statusCode !== statusCode) {
+                throw new Error(`${res.statusCode} ${res.statusMessage}`);
+            }
+
+            if (res.headers["content-type"] !== contentType) {
+                throw new Error(
+                    "Wrong content-type " + res.headers["content-type"]
+                );
+            }
+
+            resolve();
+        });
+    };
+}
+
+// DEPRECATED
+export class ResponseChecker {
+    private readonly contentType;
+    private checkCors = true;
+
+    constructor(contentType: string) {
+        this.contentType = contentType;
+    }
+
+    static forJson(): ResponseChecker {
+        return new ResponseChecker(MediaType.APPLICATION_JSON);
+    }
+
+    static forCSV(): ResponseChecker {
+        return new ResponseChecker(MediaType.TEXT_CSV);
+    }
+
+    static forGeojson(): ResponseChecker {
+        return new ResponseChecker(MediaType.APPLICATION_GEOJSON);
+    }
+
+    static forJpeg(): ResponseChecker {
+        return new ResponseChecker(MediaType.IMAGE_JPEG);
+    }
+
+    check(): CheckerFunction {
+        return this.responseChecker(() => {
+            // no need to do anything
+        });
+    }
+
+    checkJson<T>(
+        fn: (json: T, body: string, res: IncomingMessage) => void
+    ): CheckerFunction {
+        return this.responseChecker((body: string, res: IncomingMessage) => {
+            fn(JSON.parse(body), body, res);
+        });
+    }
+
+    responseChecker(
+        fn: (body: string, res: IncomingMessage) => void
+    ): CheckerFunction {
+        return async (res: IncomingMessage): Promise<void> => {
+            if (!res.statusCode) {
+                throw new Error("statusCode missing");
+            }
+
+            if (res.statusCode < 200 || res.statusCode > 299) {
+                throw new Error(`${res.statusCode} ${res.statusMessage}`);
+            }
+
+            if (this.checkCors && !res.headers["access-control-allow-origin"]) {
+                throw new Error("CORS missing");
+            }
+
+            if (res.headers["content-type"] !== this.contentType) {
+                throw new Error(
+                    "Wrong content-type " + res.headers["content-type"]
+                );
+            }
+
+            const body = await getResponseBody(res);
+
+            fn(body, res);
+        };
+    }
+}
+
+export class ContentChecker {
+    static checkJson<T>(
+        fn: (json: T, body: string, res: IncomingMessage) => void
+    ): CheckerFunction {
+        return async (res: IncomingMessage): Promise<void> => {
+            const body = await getResponseBody(res);
+
+            fn(JSON.parse(body), body, res);
+        };
+    }
+
+    static checkResponse(
+        fn: (body: string, res: IncomingMessage) => void
+    ): CheckerFunction {
+        return async (res: IncomingMessage): Promise<void> => {
+            const body = await getResponseBody(res);
+
+            fn(body, res);
+        };
+    }
+}
+
+export class ContentTypeChecker {
+    static checkContentType(contentType: MediaType) {
+        return (res: IncomingMessage) => {
+            if (!res.statusCode) {
+                throw new Error("statusCode missing");
+            }
+
+            if (res.statusCode < 200 || res.statusCode > 299) {
+                throw new Error(`${res.statusCode} ${res.statusMessage}`);
+            }
+
+            if (!res.headers["access-control-allow-origin"]) {
+                throw new Error("CORS missing");
+            }
+
+            if (res.headers["content-type"] !== contentType) {
+                throw new Error(
+                    "Wrong content-type " + res.headers["content-type"]
+                );
+            }
+        };
+    }
+}
+
+export class GeoJsonChecker {
+    static validFeatureCollection(
+        fn?: (json: FeatureCollection) => void
+    ): CheckerFunction {
+        return ResponseChecker.forGeojson().checkJson(
+            (json: FeatureCollection) => {
+                Asserter.assertEquals(json.type, "FeatureCollection");
+                Asserter.assertTrue(isValidGeoJson(json));
+
+                if (fn) {
+                    fn(json);
+                }
+            }
+        );
+    }
+}
+
+export class HeaderChecker {
+    static checkHeaderExists(headerName: string): CheckerFunction {
+        return (res: IncomingMessage) => {
+            if (!res.headers[headerName]) {
+                throw new Error("Missing header: " + headerName);
+            }
+        };
+    }
+
+    static checkHeaderMissing(headerName: string): CheckerFunction {
+        return (res: IncomingMessage) => {
+            if (res.headers[headerName]) {
+                throw new Error("Header should not exist: " + headerName);
+            }
+        };
+    }
+}