@digitraffic/common 2024.4.25-1 → 2024.6.14-1

package/README.md

@@ -4,10 +4,11 @@ This is a place for common utilities and classes that can be used in other cdk-p
 
 ## How to build
 
-Use `npm` to build the code i.e.
+Use `pnpm` to build the code i.e.
 
-    npm run build
-    npm run test
+    pnpm install
+    pnpm build
+    pnpm test
 
 ## How to use
 

package/dist/__test__/imports.test.mjs

@@ -71,10 +71,6 @@ test("dbTestutils import ok?", () => {
     const dbTestutils = import("../test/db-testutils.mjs");
     return expect(dbTestutils).resolves.toBeDefined();
 });
-test("httpserver import ok?", () => {
-    const httpserver = import("../test/httpserver.mjs");
-    return expect(httpserver).resolves.toBeDefined();
-});
 test("asserter import ok?", () => {
     const asserter = import("../test/asserter.mjs");
     return expect(asserter).resolves.toBeDefined();

package/dist/__test__/infra/acl-builder.test.mjs

@@ -1,5 +1,6 @@
 import { AclBuilder } from "../../aws/infra/acl-builder.mjs";
 import { App, Stack } from "aws-cdk-lib";
+import { expect } from "@jest/globals";
 describe("acl-builder tests", () => {
     function createBuilder() {
         const app = new App();
@@ -21,5 +22,28 @@ describe("acl-builder tests", () => {
         const acl = createBuilder().withIpRestrictionRule(["1.2.3.4", "1.2.6.6"]).build();
         expect(acl.rules).toHaveLength(1);
     });
+    test("throttle rules", () => {
+        for (const aclBuilder of [
+            createBuilder().withThrottleDigitrafficUserIp(100),
+            createBuilder().withThrottleDigitrafficUserIpAndUriPath(100),
+            createBuilder().withThrottleAnonymousUserIp(100),
+            createBuilder().withThrottleAnonymousUserIpAndUriPath(100)
+        ]) {
+            const acl = aclBuilder.build();
+            // Check that the rule exists and a custom response is defined
+            expect(acl.rules).toHaveLength(1);
+            expect(Object.keys(acl.customResponseBodies)).toHaveLength(1);
+            // Check that the rule does throttle
+            const throttleRule = acl.rules[0];
+            expect(throttleRule.statement.rateBasedStatement).toBeDefined();
+            expect(throttleRule.action.block).toBeDefined();
+        }
+    });
+    test("Cannot define two rules with the same name", () => {
+        expect(() => createBuilder()
+            .withThrottleAnonymousUserIp(10)
+            .withThrottleAnonymousUserIp(200)
+            .build()).toThrow();
+    });
 });
 //# sourceMappingURL=acl-builder.test.mjs.map

package/dist/aws/infra/acl-builder.d.mts

@@ -13,9 +13,19 @@ export declare class AclBuilder {
     readonly _rules: CfnWebACL.RuleProperty[];
     _scope: string;
     _name: string;
+    _customResponseBodies: Record<string, CfnWebACL.CustomResponseBodyProperty>;
     constructor(construct: Construct);
     isRuleDefined(rules: AWSManagedWafRule[] | "all", rule: AWSManagedWafRule): boolean;
     withAWSManagedRules(rules?: AWSManagedWafRule[] | "all"): AclBuilder;
     withIpRestrictionRule(addresses: string[]): AclBuilder;
+    withThrottleRule(name: string, priority: number, limit: number, customResponseBodyKey: string, isHeaderRequired: boolean, isBasedOnIpAndUriPath: boolean): AclBuilder;
+    withCustomResponseBody(key: string, customResponseBody: CfnWebACL.CustomResponseBodyProperty): this;
+    withThrottleDigitrafficUserIp(limit: number): AclBuilder;
+    withThrottleDigitrafficUserIpAndUriPath(limit: number): AclBuilder;
+    withThrottleAnonymousUserIp(limit: number): AclBuilder;
+    withThrottleAnonymousUserIpAndUriPath(limit: number): AclBuilder;
+    _isCustomResponseBodyKeySet(key: string): boolean;
+    _addThrottleResponseBody(customResponseBodyKey: string, limit: number): void;
+    _logMissingLimit(method: string): void;
     build(): CfnWebACL;
 }

package/dist/aws/infra/acl-builder.mjs

@@ -1,4 +1,5 @@
 import { CfnIPSet, CfnWebACL } from "aws-cdk-lib/aws-wafv2";
+import { logger } from "../runtime/dt-logger-default.mjs";
 /**
  * Builder class for building CfnWebACL.
  *
@@ -11,6 +12,7 @@ export class AclBuilder {
     _rules = [];
     _scope = "CLOUDFRONT";
     _name = "WebACL";
+    _customResponseBodies = {};
     constructor(construct) {
         this._construct = construct;
     }
@@ -55,10 +57,98 @@ export class AclBuilder {
         });
         return this;
     }
+    withThrottleRule(name, priority, limit, customResponseBodyKey, isHeaderRequired, isBasedOnIpAndUriPath) {
+        this._rules.push({
+            name,
+            priority,
+            visibilityConfig: {
+                sampledRequestsEnabled: true,
+                cloudWatchMetricsEnabled: true,
+                metricName: name
+            },
+            action: {
+                block: {
+                    customResponse: {
+                        responseCode: 429,
+                        customResponseBodyKey
+                    }
+                }
+            },
+            statement: createThrottleStatement(limit, isHeaderRequired, isBasedOnIpAndUriPath)
+        });
+        return this;
+    }
+    withCustomResponseBody(key, customResponseBody) {
+        if (key in this._customResponseBodies) {
+            logger.warn({
+                method: "acl-builder.withCustomResponseBody",
+                message: `Overriding custom response body with key ${key}`
+            });
+        }
+        this._customResponseBodies[key] = customResponseBody;
+        return this;
+    }
+    withThrottleDigitrafficUserIp(limit) {
+        if (limit == null) {
+            this._logMissingLimit("withThrottleDigitrafficUserIp");
+            return this;
+        }
+        const customResponseBodyKey = `IP_THROTTLE_DIGITRAFFIC_USER_${limit}`;
+        this._addThrottleResponseBody(customResponseBodyKey, limit);
+        return this.withThrottleRule("ThrottleRuleWithDigitrafficUser", 1, limit, customResponseBodyKey, true, false);
+    }
+    withThrottleDigitrafficUserIpAndUriPath(limit) {
+        if (limit == null) {
+            this._logMissingLimit("withThrottleDigitrafficUserIpAndUriPath");
+            return this;
+        }
+        const customResponseBodyKey = `IP_PATH_THROTTLE_DIGITRAFFIC_USER_${limit}`;
+        this._addThrottleResponseBody(customResponseBodyKey, limit);
+        return this.withThrottleRule("ThrottleRuleIPQueryWithDigitrafficUser", 2, limit, customResponseBodyKey, true, true);
+    }
+    withThrottleAnonymousUserIp(limit) {
+        if (limit == null) {
+            this._logMissingLimit("withThrottleAnonymousUserIp");
+            return this;
+        }
+        const customResponseBodyKey = `IP_THROTTLE_ANONYMOUS_USER_${limit}`;
+        this._addThrottleResponseBody(customResponseBodyKey, limit);
+        return this.withThrottleRule("ThrottleRuleWithAnonymousUser", 3, limit, customResponseBodyKey, false, false);
+    }
+    withThrottleAnonymousUserIpAndUriPath(limit) {
+        if (limit == null) {
+            this._logMissingLimit("withThrottleAnonymousUserIpAndUriPath");
+            return this;
+        }
+        const customResponseBodyKey = `IP_PATH_THROTTLE_ANONYMOUS_USER_${limit}`;
+        this._addThrottleResponseBody(customResponseBodyKey, limit);
+        return this.withThrottleRule("ThrottleRuleIPQueryWithAnonymousUser", 4, limit, customResponseBodyKey, false, true);
+    }
+    _isCustomResponseBodyKeySet(key) {
+        return key in this._customResponseBodies;
+    }
+    _addThrottleResponseBody(customResponseBodyKey, limit) {
+        if (!this._isCustomResponseBodyKeySet(customResponseBodyKey)) {
+            this.withCustomResponseBody(customResponseBodyKey, {
+                content: `Request rate is limited to ${limit} requests in a 5 minute window.`,
+                contentType: "TEXT_PLAIN"
+            });
+        }
+    }
+    _logMissingLimit(method) {
+        logger.warn({
+            method: `acl-builder.${method}`,
+            message: `'limit' was not defined. Not setting a throttle rule`
+        });
+    }
     build() {
         if (this._rules.length === 0) {
             throw new Error("No rules defined for WebACL");
         }
+        const uniqueRuleNames = new Set(this._rules.map(rule => rule.name));
+        if (uniqueRuleNames.size != this._rules.length) {
+            throw new Error("Tried to create an Access Control List with multiple rules having the same name");
+        }
         const acl = new CfnWebACL(this._construct, this._name, {
             defaultAction: { allow: {} },
             scope: this._scope,
@@ -68,11 +158,75 @@ export class AclBuilder {
                 sampledRequestsEnabled: false
             },
             rules: this._rules,
-            //            customResponseBodies
+            customResponseBodies: this._customResponseBodies,
         });
         return acl;
     }
 }
+const CUSTOM_KEYS_IP_AND_URI_PATH = [
+    {
+        uriPath: {
+            textTransformations: [
+                {
+                    priority: 1,
+                    type: "LOWERCASE"
+                },
+                {
+                    priority: 2,
+                    type: "NORMALIZE_PATH"
+                },
+                {
+                    priority: 3,
+                    type: "MD5"
+                }
+            ]
+        }
+    },
+    {
+        ip: {}
+    }
+];
+function notStatement(statement) {
+    return {
+        notStatement: {
+            statement
+        }
+    };
+}
+function createThrottleStatement(limit, isHeaderRequired, isBasedOnIpAndUriPath) {
+    // this statement matches empty digitraffic-user -header
+    const matchStatement = {
+        sizeConstraintStatement: {
+            comparisonOperator: isHeaderRequired ? "GT" : "GE",
+            fieldToMatch: {
+                singleHeader: {
+                    Name: "digitraffic-user"
+                }
+            },
+            textTransformations: [{ priority: 0, type: "NONE" }],
+            size: 0
+        }
+    };
+    // header present       -> size > 0
+    // header not present   -> NOT(size >= 0)
+    if (isBasedOnIpAndUriPath) {
+        return {
+            rateBasedStatement: {
+                aggregateKeyType: "CUSTOM_KEYS",
+                customKeys: CUSTOM_KEYS_IP_AND_URI_PATH,
+                limit: limit,
+                scopeDownStatement: isHeaderRequired ? matchStatement : notStatement(matchStatement)
+            }
+        };
+    }
+    return {
+        rateBasedStatement: {
+            aggregateKeyType: "IP",
+            limit: limit,
+            scopeDownStatement: isHeaderRequired ? matchStatement : notStatement(matchStatement)
+        }
+    };
+}
 function createAWSCommonRuleSet() {
     return createRuleProperty("AWS-AWSManagedRulesCommonRuleSet", 70, {
         statement: {

package/dist/aws/infra/stack/monitoredfunction.mjs

@@ -1,4 +1,4 @@
-import { Function, LoggingFormat } from "aws-cdk-lib/aws-lambda";
+import { ApplicationLogLevel, Function, LoggingFormat, SystemLogLevel } from "aws-cdk-lib/aws-lambda";
 import { Stack } from "aws-cdk-lib";
 import { SnsAction } from "aws-cdk-lib/aws-cloudwatch-actions";
 import { ComparisonOperator, Metric } from "aws-cdk-lib/aws-cloudwatch";
@@ -75,7 +75,13 @@ export class MonitoredFunction extends Function {
      */
     constructor(scope, id, functionProps, alarmSnsTopic, warningSnsTopic, production, trafficType, props) {
         // Set default loggingFormat to JSON if not explicitly set to TEXT
-        super(scope, id, { ...{ loggingFormat: LoggingFormat.JSON }, ...functionProps });
+        super(scope, id, {
+            ...{
+                loggingFormat: LoggingFormat.JSON,
+                applicationLogLevel: ApplicationLogLevel.DEBUG,
+                systemLogLevel: SystemLogLevel.INFO
+            }, ...functionProps
+        });
         if (functionProps.functionName === undefined) {
             throw new Error("Function name not provided");
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@digitraffic/common",
-  "version": "2024.4.25-1",
+  "version": "2024.06.14-1",
   "description": "",
   "type": "module",
   "repository": {

package/dist/__test__/test/httpserver.test.d.mts

@@ -1 +0,0 @@
-export {};

package/dist/__test__/test/httpserver.test.mjs

@@ -1,154 +0,0 @@
-import { TestHttpServer, ERROR_NO_MATCH, ERRORCODE_NOT_FOUND, } from "../../test/httpserver.mjs";
-import { IncomingMessage } from "http";
-import { Socket } from "net";
-import { AsyncLocalStorage } from "node:async_hooks";
-import * as http from "http";
-import { expect } from "@jest/globals";
-const threadLocalPort = new AsyncLocalStorage();
-const DEFAULT_PATH = "/";
-const DEFAULT_PROPS = {
-    "/": () => "",
-};
-const findOpenPort = async (excludedPorts) => {
-    const ephemeralPorts = Array.from({ length: 65535 - 1024 + 1 }, (__, i) => 1024 + i);
-    const allSocketEvents = [
-        "close",
-        "connect",
-        "data",
-        "drain",
-        "end",
-        "error",
-        "lookup",
-        "ready",
-        "timeout",
-    ];
-    let openPort = null;
-    for (const testPort of ephemeralPorts) {
-        if (openPort !== null) {
-            break;
-        }
-        if (excludedPorts.has(testPort)) {
-            continue;
-        }
-        const portConnected = new Promise((resolve) => {
-            const socket = new Socket();
-            socket.setTimeout(500);
-            for (const socketEvent of allSocketEvents) {
-                if (socketEvent === "error") {
-                    socket.on(socketEvent, (error) => {
-                        socket.destroy();
-                        if (error.code === "ECONNREFUSED") {
-                            resolve(testPort);
-                        }
-                        else {
-                            resolve(null);
-                        }
-                    });
-                }
-                else {
-                    socket.on(socketEvent, () => {
-                        socket.destroy();
-                        resolve(null);
-                    });
-                }
-            }
-            // connect method is asynchronous. That is why we wrap this thing inside of a promise.
-            socket.connect({ port: testPort, host: "127.0.0.1" });
-        });
-        openPort = await portConnected;
-    }
-    if (openPort === null) {
-        throw Error("All ephemeral ports in use!");
-    }
-    return openPort;
-};
-const usedPorts = new Set();
-async function withServer(fn, props = DEFAULT_PROPS, statusCode = 200) {
-    const server = new TestHttpServer();
-    let openPort;
-    while (!openPort) {
-        const foundPort = await findOpenPort(usedPorts);
-        console.info(`foundPort ${foundPort}`);
-        if (!usedPorts.has(foundPort)) {
-            usedPorts.add(foundPort);
-            openPort = foundPort;
-        }
-    }
-    console.info(`Using port ${openPort} to run the test`);
-    server.listen(openPort, props, false, statusCode);
-    threadLocalPort.enterWith(openPort);
-    try {
-        await fn(server);
-    }
-    finally {
-        await server.close();
-        console.info("Server closed");
-    }
-}
-function sendGetRequest(path = DEFAULT_PATH) {
-    return sendRequest("GET", path);
-}
-function sendPostRequest(path = DEFAULT_PATH, body) {
-    return sendRequest("POST", path, body);
-}
-function sendRequest(method, path, body) {
-    return new Promise((resolve, reject) => {
-        const port = threadLocalPort.getStore();
-        const request = http.request({
-            path,
-            port,
-            method,
-        }, (response) => {
-            response.on("data", () => {
-                // do nothing
-            });
-            //the whole response has been received, so we just print it out here
-            response.on("end", () => {
-                resolve(response);
-            });
-            response.on("error", (error) => {
-                reject(error);
-            });
-        });
-        if (method === "POST") {
-            request.write(body);
-        }
-        request.end();
-    });
-}
-test("no calls", () => {
-    return withServer((server) => {
-        expect(server.getCallCount()).toEqual(0);
-    });
-});
-test("one get", async () => {
-    await withServer(async (server) => {
-        await sendGetRequest();
-        expect(server.getCallCount()).toEqual(1);
-    });
-});
-test("one get - no MATCH", async () => {
-    await withServer(async (server) => {
-        const response = await sendGetRequest("/no-match");
-        expect(server.getCallCount()).toEqual(1);
-        expect(server.getRequestBody(0)).toEqual(ERROR_NO_MATCH);
-        expect(response.statusCode).toEqual(ERRORCODE_NOT_FOUND);
-    });
-});
-test("get - error 405", async () => {
-    const ERROR_CODE = 405;
-    await withServer(async (server) => {
-        const response = await sendGetRequest();
-        expect(server.getCallCount()).toEqual(1);
-        expect(response.statusCode).toEqual(ERROR_CODE);
-    }, DEFAULT_PROPS, ERROR_CODE);
-});
-test("one post", async () => {
-    await withServer(async (server) => {
-        const testBody = "Testing123!";
-        await sendPostRequest(DEFAULT_PATH, testBody);
-        expect(server.getCallCount()).toEqual(1);
-        expect(server.getRequestBody(0)).toEqual(testBody);
-    });
-});
-//# sourceMappingURL=httpserver.test.mjs.map

package/dist/test/httpserver.d.mts

@@ -1,17 +0,0 @@
-export declare const ERROR_NO_MATCH = "NO MATCH";
-export declare const ERRORCODE_NOT_FOUND = 404;
-/**
- * A mock HTTP server created for testing connections from a Lambda to an outside integration
- */
-export declare class TestHttpServer {
-    private server?;
-    private debug;
-    private messageStack;
-    constructor();
-    getCallCount(): number;
-    getRequestBody(callNumber: number): string;
-    listen(port: number, props: ListenProperties, debug?: boolean, statusCode?: number): void;
-    close(): Promise<boolean>;
-    private debuglog;
-}
-export type ListenProperties = Record<string, (url?: string, data?: string) => string>;

package/dist/test/httpserver.mjs

@@ -1,85 +0,0 @@
-import { Server, createServer } from "http";
-import { parse } from "url";
-export const ERROR_NO_MATCH = "NO MATCH";
-export const ERRORCODE_NOT_FOUND = 404;
-/**
- * A mock HTTP server created for testing connections from a Lambda to an outside integration
- */
-export class TestHttpServer {
-    server;
-    debug;
-    messageStack;
-    constructor() {
-        this.debug = false;
-        this.messageStack = [];
-    }
-    getCallCount() {
-        return this.messageStack.length;
-    }
-    getRequestBody(callNumber) {
-        return this.messageStack[callNumber] ?? '';
-    }
-    listen(port, props, debug = false, statusCode = 200) {
-        this.debug = debug;
-        this.messageStack = [];
-        this.debuglog(`Starting test server on port ${port}`);
-        this.server = createServer((req, res) => {
-            this.debuglog("Mapped urls: ");
-            Object.keys(props).forEach((k) => this.debuglog(k));
-            if (!req.url) {
-                throw new Error("Missing request url!");
-            }
-            this.debuglog(`Received request to url ${req.url} ..`);
-            const path = parse(req.url).pathname;
-            if (!path) {
-                throw new Error("Missing path from request!");
-            }
-            let dataStr = "";
-            req.on("data", (chunk) => {
-                if (chunk) {
-                    dataStr += chunk;
-                }
-            });
-            if (path in props) {
-                this.debuglog("..url matched");
-                res.setHeader("Access-Control-Allow-Origin", "*");
-                res.setHeader("Access-Control-Allow-Headers", "Authorization,X-User-Id,X-Auth-Token");
-                res.writeHead(statusCode);
-                req.on("end", () => {
-                    // assume sent data is in JSON format
-                    this.messageStack[this.messageStack.length] = dataStr;
-                    const invokable = props[path];
-                    res.end(invokable(req.url, dataStr));
-                });
-            }
-            else {
-                this.debuglog(`..no match for ${path}`);
-                req.on("end", () => {
-                    // assume sent data is in JSON format
-                    this.messageStack[this.messageStack.length] =
-                        ERROR_NO_MATCH;
-                    res.writeHead(ERRORCODE_NOT_FOUND);
-                    res.end(ERROR_NO_MATCH);
-                });
-            }
-        });
-        this.server.listen(port);
-    }
-    close() {
-        return new Promise((resolve, reject) => {
-            this.debuglog("Closing test server");
-            if (this.server !== undefined) {
-                this.server.close((error) => error != null ? reject(false) : resolve(true));
-            }
-            else {
-                resolve(true);
-            }
-        });
-    }
-    debuglog(str) {
-        if (this.debug) {
-            console.debug(str);
-        }
-    }
-}
-//# sourceMappingURL=httpserver.mjs.map