cdk-nuxt 2.0.0 → 2.1.0

package/lib/functions/access-logs-analysis/partitioning/.gitignore

@@ -0,0 +1,2 @@
+node_modules
+build

package/lib/functions/access-logs-analysis/partitioning/build/app/create-partition.d.ts

@@ -0,0 +1 @@
+export declare const handler: () => Promise<void>;

package/lib/functions/access-logs-analysis/partitioning/build/app/create-partition.js

@@ -0,0 +1,57 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handler = void 0;
+/**
+ * This script creates a partition for the upcoming hour.
+ * Taken and adjusted from https://github.com/aws-samples/amazon-cloudfront-access-logs-queries/blob/mainline/functions/createPartitions.js.
+ */
+// noinspection DuplicatedCode
+const client_athena_1 = require("@aws-sdk/client-athena");
+const util_1 = require("./util");
+const workgroup = process.env.WORKGROUP;
+if (!workgroup) {
+    throw new Error('Required environment variable WORKGROUP missing!');
+}
+const database = process.env.DATABASE;
+if (!database) {
+    throw new Error('Required environment variable DATABASE missing!');
+}
+const table = process.env.TABLE;
+if (!table) {
+    throw new Error('Required environment variable TABLE missing!');
+}
+const internalHandler = async () => {
+    try {
+        const nextHour = new Date(Date.now() + 60 * 60 * 1000);
+        const year = nextHour.getUTCFullYear();
+        const month = (nextHour.getUTCMonth() + 1).toString().padStart(2, '0');
+        const day = nextHour
+            .getUTCDate()
+            .toString()
+            .padStart(2, '0');
+        const hour = nextHour
+            .getUTCHours()
+            .toString()
+            .padStart(2, '0');
+        console.log('creating partition', { year, month, day, hour });
+        const command = new client_athena_1.StartQueryExecutionCommand({
+            QueryString: `
+        ALTER TABLE ${database}.${table}
+        ADD IF NOT EXISTS
+        PARTITION (
+            year = '${year}',
+            month = '${month}',
+            day = '${day}',
+            hour = '${hour}' );`,
+            WorkGroup: workgroup,
+            QueryExecutionContext: { Database: database },
+        });
+        await (0, util_1.executeAndAwaitResponse)(command, true);
+        console.log('partition successfully created');
+    }
+    catch (error) {
+        console.error('### unexpected runtime error ###', error);
+    }
+};
+exports.handler = internalHandler;
+//# sourceMappingURL=create-partition.js.map

package/lib/functions/access-logs-analysis/partitioning/build/app/create-partition.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"create-partition.js","sourceRoot":"/","sources":["create-partition.ts"],"names":[],"mappings":";;;AAAA;;;GAGG;AACH,8BAA8B;AAC9B,0DAAkE;AAClE,iCAA+C;AAE/C,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;AACxC,IAAI,CAAC,SAAS,EAAE;IACd,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;CACrE;AACD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;AACtC,IAAI,CAAC,QAAQ,EAAE;IACb,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;CACpE;AACD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC;AAChC,IAAI,CAAC,KAAK,EAAE;IACV,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;CACjE;AAED,MAAM,eAAe,GAAG,KAAK,IAAI,EAAE;IACjC,IAAI;QACF,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;QACvD,MAAM,IAAI,GAAG,QAAQ,CAAC,cAAc,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACvE,MAAM,GAAG,GAAG,QAAQ;aACjB,UAAU,EAAE;aACZ,QAAQ,EAAE;aACV,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACpB,MAAM,IAAI,GAAG,QAAQ;aAClB,WAAW,EAAE;aACb,QAAQ,EAAE;aACV,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAEpB,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,EAAC,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAC,CAAC,CAAC;QAE5D,MAAM,OAAO,GAAG,IAAI,0CAA0B,CAAC;YAC7C,WAAW,EAAE;sBACG,QAAQ,IAAI,KAAK;;;sBAGjB,IAAI;uBACH,KAAK;qBACP,GAAG;sBACF,IAAI,MAAM;YAC1B,SAAS,EAAE,SAAS;YACpB,qBAAqB,EAAE,EAAC,QAAQ,EAAE,QAAQ,EAAC;SAC5C,CAAC,CAAC;QAEH,MAAM,IAAA,8BAAuB,EAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;KAC/C;IAAC,OAAO,KAAK,EAAE;QACd,OAAO,CAAC,KAAK,CAAC,kCAAkC,EAAE,KAAK,CAAC,CAAC;KAC1D;AACH,CAAC,CAAC;AAEW,QAAA,OAAO,GAAG,eAAe,CAAC","sourcesContent":["/**\n * This script creates a partition for the upcoming hour.\n * Taken and adjusted from https://github.com/aws-samples/amazon-cloudfront-access-logs-queries/blob/mainline/functions/createPartitions.js.\n */\n// noinspection DuplicatedCode\nimport {StartQueryExecutionCommand} from '@aws-sdk/client-athena';\nimport {executeAndAwaitResponse} from './util';\n\nconst workgroup = process.env.WORKGROUP;\nif (!workgroup) {\n  throw new Error('Required environment variable WORKGROUP missing!');\n}\nconst database = process.env.DATABASE;\nif (!database) {\n  throw new Error('Required environment variable DATABASE missing!');\n}\nconst table = process.env.TABLE;\nif (!table) {\n  throw new Error('Required environment variable TABLE missing!');\n}\n\nconst internalHandler = async () => {\n  try {\n    const nextHour = new Date(Date.now() + 60 * 60 * 1000);\n    const year = nextHour.getUTCFullYear();\n    const month = (nextHour.getUTCMonth() + 1).toString().padStart(2, '0');\n    const day = nextHour\n      .getUTCDate()\n      .toString()\n      .padStart(2, '0');\n    const hour = nextHour\n      .getUTCHours()\n      .toString()\n      .padStart(2, '0');\n\n    console.log('creating partition', {year, month, day, hour});\n\n    const command = new StartQueryExecutionCommand({\n      QueryString: `\n        ALTER TABLE ${database}.${table}\n        ADD IF NOT EXISTS\n        PARTITION (\n            year = '${year}',\n            month = '${month}',\n            day = '${day}',\n            hour = '${hour}' );`,\n      WorkGroup: workgroup,\n      QueryExecutionContext: {Database: database},\n    });\n\n    await executeAndAwaitResponse(command, true);\n    console.log('partition successfully created');\n  } catch (error) {\n    console.error('### unexpected runtime error ###', error);\n  }\n};\n\nexport const handler = internalHandler;\n"]}

package/lib/functions/access-logs-analysis/partitioning/build/app/transform-partition.d.ts

@@ -0,0 +1,2 @@
+import type { TransformPartitionEvent } from './types';
+export declare const handler: (event: TransformPartitionEvent) => Promise<void>;

package/lib/functions/access-logs-analysis/partitioning/build/app/transform-partition.js

@@ -0,0 +1,72 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handler = void 0;
+/**
+ * This script transforms the access log partition of two hours ago into the Apache parquet format.
+ * Taken and adjusted from https://github.com/aws-samples/amazon-cloudfront-access-logs-queries/blob/mainline/functions/transformPartition.js.
+ */
+const client_athena_1 = require("@aws-sdk/client-athena");
+const util_1 = require("./util");
+const workgroup = process.env.WORKGROUP;
+if (!workgroup) {
+    throw new Error('Required environment variable WORKGROUP missing!');
+}
+const database = process.env.DATABASE;
+if (!database) {
+    throw new Error('Required environment variable DATABASE missing!');
+}
+const sourceTable = process.env.SOURCE_TABLE;
+if (!sourceTable) {
+    throw new Error('Required environment variable SOURCE_TABLE missing!');
+}
+const targetTable = process.env.TARGET_TABLE;
+if (!targetTable) {
+    throw new Error('Required environment variable TARGET_TABLE missing!');
+}
+/**
+ * Reducer, that computes the column statements for the insert command. Columns are either selected by their name, or
+ * are computed by an expression specified in `columnTransformations`.
+ */
+const buildQueryColumns = (columnTransformations, previous, current) => {
+    const columnExpression = Object.keys(columnTransformations).includes(current)
+        ? `${columnTransformations[current]} AS ${current}`
+        : current;
+    return previous.length === 0 ? columnExpression : `${previous}, ${columnExpression}`;
+};
+/**
+ * Entrypoint
+ */
+const internalHandler = async (event) => {
+    try {
+        const partitionHour = new Date(Date.now() - 120 * 60 * 1000);
+        const year = partitionHour.getUTCFullYear();
+        const month = (partitionHour.getUTCMonth() + 1).toString().padStart(2, '0');
+        const day = partitionHour.getUTCDate().toString().padStart(2, '0');
+        const hour = partitionHour.getUTCHours().toString().padStart(2, '0');
+        console.log('transforming partition', { year, month, day, hour });
+        // apply transformations on certain columns:
+        // - obfuscate IPs
+        // - optionally filter cookies
+        const selectExpr = event.columnNames.reduce((previous, current) => buildQueryColumns(event.columnTransformations, previous, current), '');
+        const command = new client_athena_1.StartQueryExecutionCommand({
+            QueryString: `
+        INSERT INTO ${database}.${targetTable} (${event.columnNames.join(',')})
+        SELECT ${selectExpr}
+        FROM ${database}.${sourceTable}
+        WHERE year = '${year}'
+          AND month = '${month}'
+          AND day = '${day}'
+          AND hour = '${hour}';`,
+            WorkGroup: workgroup,
+            QueryExecutionContext: { Database: database },
+        });
+        if (await (0, util_1.executeAndAwaitResponse)(command, false)) {
+            console.log('successfully transformed partition', { year, month, day, hour });
+        }
+    }
+    catch (error) {
+        console.error('### unexpected runtime error ###', error);
+    }
+};
+exports.handler = internalHandler;
+//# sourceMappingURL=transform-partition.js.map

package/lib/functions/access-logs-analysis/partitioning/build/app/transform-partition.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"transform-partition.js","sourceRoot":"/","sources":["transform-partition.ts"],"names":[],"mappings":";;;AAAA;;;GAGG;AACH,0DAAkE;AAClE,iCAA+C;AAG/C,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;AACxC,IAAI,CAAC,SAAS,EAAE;IACd,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;CACrE;AACD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;AACtC,IAAI,CAAC,QAAQ,EAAE;IACb,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;CACpE;AACD,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;AAC7C,IAAI,CAAC,WAAW,EAAE;IAChB,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;CACxE;AACD,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;AAC7C,IAAI,CAAC,WAAW,EAAE;IAChB,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;CACxE;AAED;;;GAGG;AACH,MAAM,iBAAiB,GAAG,CACxB,qBAAgD,EAChD,QAAgB,EAChB,OAAe,EACP,EAAE;IACV,MAAM,gBAAgB,GAAG,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC;QAC3E,CAAC,CAAC,GAAG,qBAAqB,CAAC,OAAO,CAAC,OAAO,OAAO,EAAE;QACnD,CAAC,CAAC,OAAO,CAAC;IACZ,OAAO,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,GAAG,QAAQ,KAAK,gBAAgB,EAAE,CAAC;AACvF,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,eAAe,GAAG,KAAK,EAAE,KAA8B,EAAE,EAAE;IAC/D,IAAI;QACF,MAAM,aAAa,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAG,aAAa,CAAC,cAAc,EAAE,CAAC;QAC5C,MAAM,KAAK,GAAG,CAAC,aAAa,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC5E,MAAM,GAAG,GAAG,aAAa,CAAC,UAAU,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACnE,MAAM,IAAI,GAAG,aAAa,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAErE,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,EAAC,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAC,CAAC,CAAC;QAEhE,4CAA4C;QAC5C,kBAAkB;QAClB,8BAA8B;QAC9B,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CACzC,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,CAAC,iBAAiB,CAAC,KAAK,CAAC,qBAAqB,EAAE,QAAQ,EAAE,OAAO,CAAC,EACxF,EAAE,CACH,CAAC;QAEF,MAAM,OAAO,GAAG,IAAI,0CAA0B,CAAC;YAC7C,WAAW,EAAE;sBACG,QAAQ,IAAI,WAAW,KAAK,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC;iBAC5D,UAAU;eACZ,QAAQ,IAAI,WAAW;wBACd,IAAI;yBACH,KAAK;uBACP,GAAG;wBACF,IAAI,IAAI;YAC1B,SAAS,EAAE,SAAS;YACpB,qBAAqB,EAAE,EAAC,QAAQ,EAAE,QAAQ,EAAC;SAC5C,CAAC,CAAC;QAEH,IAAI,MAAM,IAAA,8BAAuB,EAAC,OAAO,EAAE,KAAK,CAAC,EAAE;YACjD,OAAO,CAAC,GAAG,CAAC,oCAAoC,EAAE,EAAC,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAC,CAAC,CAAC;SAC7E;KACF;IAAC,OAAO,KAAK,EAAE;QACd,OAAO,CAAC,KAAK,CAAC,kCAAkC,EAAE,KAAK,CAAC,CAAC;KAC1D;AACH,CAAC,CAAC;AAEW,QAAA,OAAO,GAAG,eAAe,CAAC","sourcesContent":["/**\n * This script transforms the access log partition of two hours ago into the Apache parquet format.\n * Taken and adjusted from https://github.com/aws-samples/amazon-cloudfront-access-logs-queries/blob/mainline/functions/transformPartition.js.\n */\nimport {StartQueryExecutionCommand} from '@aws-sdk/client-athena';\nimport {executeAndAwaitResponse} from './util';\nimport type {ColumnTransformationRules, TransformPartitionEvent} from './types';\n\nconst workgroup = process.env.WORKGROUP;\nif (!workgroup) {\n  throw new Error('Required environment variable WORKGROUP missing!');\n}\nconst database = process.env.DATABASE;\nif (!database) {\n  throw new Error('Required environment variable DATABASE missing!');\n}\nconst sourceTable = process.env.SOURCE_TABLE;\nif (!sourceTable) {\n  throw new Error('Required environment variable SOURCE_TABLE missing!');\n}\nconst targetTable = process.env.TARGET_TABLE;\nif (!targetTable) {\n  throw new Error('Required environment variable TARGET_TABLE missing!');\n}\n\n/**\n * Reducer, that computes the column statements for the insert command. Columns are either selected by their name, or\n * are computed by an expression specified in `columnTransformations`.\n */\nconst buildQueryColumns = (\n  columnTransformations: ColumnTransformationRules,\n  previous: string,\n  current: string\n): string => {\n  const columnExpression = Object.keys(columnTransformations).includes(current)\n    ? `${columnTransformations[current]} AS ${current}`\n    : current;\n  return previous.length === 0 ? columnExpression : `${previous}, ${columnExpression}`;\n};\n\n/**\n * Entrypoint\n */\nconst internalHandler = async (event: TransformPartitionEvent) => {\n  try {\n    const partitionHour = new Date(Date.now() - 120 * 60 * 1000);\n    const year = partitionHour.getUTCFullYear();\n    const month = (partitionHour.getUTCMonth() + 1).toString().padStart(2, '0');\n    const day = partitionHour.getUTCDate().toString().padStart(2, '0');\n    const hour = partitionHour.getUTCHours().toString().padStart(2, '0');\n\n    console.log('transforming partition', {year, month, day, hour});\n\n    // apply transformations on certain columns:\n    // - obfuscate IPs\n    // - optionally filter cookies\n    const selectExpr = event.columnNames.reduce(\n      (previous, current) => buildQueryColumns(event.columnTransformations, previous, current),\n      ''\n    );\n\n    const command = new StartQueryExecutionCommand({\n      QueryString: `\n        INSERT INTO ${database}.${targetTable} (${event.columnNames.join(',')})\n        SELECT ${selectExpr}\n        FROM ${database}.${sourceTable}\n        WHERE year = '${year}'\n          AND month = '${month}'\n          AND day = '${day}'\n          AND hour = '${hour}';`,\n      WorkGroup: workgroup,\n      QueryExecutionContext: {Database: database},\n    });\n\n    if (await executeAndAwaitResponse(command, false)) {\n      console.log('successfully transformed partition', {year, month, day, hour});\n    }\n  } catch (error) {\n    console.error('### unexpected runtime error ###', error);\n  }\n};\n\nexport const handler = internalHandler;\n"]}

package/lib/functions/access-logs-analysis/partitioning/build/app/types.d.ts

@@ -0,0 +1,7 @@
+export interface ColumnTransformationRules {
+    readonly [columnName: string]: string | undefined;
+}
+export interface TransformPartitionEvent {
+    readonly columnNames: string[];
+    readonly columnTransformations: ColumnTransformationRules;
+}

package/lib/functions/access-logs-analysis/partitioning/build/app/types.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/lib/functions/access-logs-analysis/partitioning/build/app/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"/","sources":["types.ts"],"names":[],"mappings":"","sourcesContent":["export interface ColumnTransformationRules {\n  readonly [columnName: string]: string | undefined;\n}\n\nexport interface TransformPartitionEvent {\n  readonly columnNames: string[];\n  readonly columnTransformations: ColumnTransformationRules;\n}\n"]}

package/lib/functions/access-logs-analysis/partitioning/build/app/util.d.ts

@@ -0,0 +1,9 @@
+import { StartQueryExecutionCommand } from '@aws-sdk/client-athena';
+/**
+ * Executes the given command and waits up to 10s for the command to complete.
+ * @param failOnTimeout <code>true</code> in order to throw an exception,
+ * or <code>false</code> to just log the error to the console
+ * @return <code>true</code> on success or <code>false</code> if the command timed out and
+ * <code>failOnTimeout</code> was set to false
+ */
+export declare const executeAndAwaitResponse: (command: StartQueryExecutionCommand, failOnTimeout: boolean) => Promise<boolean>;

package/lib/functions/access-logs-analysis/partitioning/build/app/util.js

@@ -0,0 +1,44 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.executeAndAwaitResponse = void 0;
+const client_athena_1 = require("@aws-sdk/client-athena");
+/**
+ * Executes the given command and waits up to 10s for the command to complete.
+ * @param failOnTimeout <code>true</code> in order to throw an exception,
+ * or <code>false</code> to just log the error to the console
+ * @return <code>true</code> on success or <code>false</code> if the command timed out and
+ * <code>failOnTimeout</code> was set to false
+ */
+const executeAndAwaitResponse = async (command, failOnTimeout) => {
+    console.log('executing command ', command.input.QueryString, '...');
+    const athena = new client_athena_1.AthenaClient({});
+    const response = await athena.send(command);
+    // wait up to 10s for completion
+    const checkStatus = new client_athena_1.GetQueryExecutionCommand({
+        QueryExecutionId: response.QueryExecutionId,
+    });
+    for (let i = 1; i <= 50; i++) {
+        const state = await athena.send(checkStatus);
+        switch (state.QueryExecution.Status.State) {
+            case client_athena_1.QueryExecutionState.SUCCEEDED:
+                console.log('command completed successfully');
+                return true;
+            case client_athena_1.QueryExecutionState.CANCELLED:
+            case client_athena_1.QueryExecutionState.FAILED:
+                // noinspection TypeScriptUnresolvedFunction
+                throw new Error('Command execution failed! Query:\n' + command.input.QueryString);
+        }
+        await new Promise(resolve => setTimeout(resolve, 200));
+    }
+    const errorMessage = 'Status of partition creation unknown - waited for 10s but command did not complete';
+    if (failOnTimeout) {
+        // noinspection TypeScriptUnresolvedFunction
+        throw new Error(errorMessage);
+    }
+    else {
+        console.error(errorMessage);
+    }
+    return false;
+};
+exports.executeAndAwaitResponse = executeAndAwaitResponse;
+//# sourceMappingURL=util.js.map

package/lib/functions/access-logs-analysis/partitioning/build/app/util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.js","sourceRoot":"/","sources":["util.ts"],"names":[],"mappings":";;;AAAA,0DAKgC;AAEhC;;;;;;GAMG;AACI,MAAM,uBAAuB,GAAG,KAAK,EAC1C,OAAmC,EACnC,aAAsB,EACJ,EAAE;IACpB,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;IAEpE,MAAM,MAAM,GAAG,IAAI,4BAAY,CAAC,EAAE,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAE5C,gCAAgC;IAChC,MAAM,WAAW,GAAG,IAAI,wCAAwB,CAAC;QAC/C,gBAAgB,EAAE,QAAQ,CAAC,gBAAgB;KAC5C,CAAC,CAAC;IAEH,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE;QAC5B,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC7C,QAAQ,KAAK,CAAC,cAAe,CAAC,MAAO,CAAC,KAAK,EAAE;YAC3C,KAAK,mCAAmB,CAAC,SAAS;gBAChC,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;gBAC9C,OAAO,IAAI,CAAC;YACd,KAAK,mCAAmB,CAAC,SAAS,CAAC;YACnC,KAAK,mCAAmB,CAAC,MAAM;gBAC7B,4CAA4C;gBAC5C,MAAM,IAAI,KAAK,CAAC,oCAAoC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;SACrF;QAED,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;KACxD;IAED,MAAM,YAAY,GAAG,oFAAoF,CAAC;IAC1G,IAAI,aAAa,EAAE;QACjB,4CAA4C;QAC5C,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC;KAC/B;SAAM;QACL,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;KAC7B;IACD,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AArCW,QAAA,uBAAuB,2BAqClC","sourcesContent":["import {\n  AthenaClient,\n  GetQueryExecutionCommand,\n  QueryExecutionState,\n  StartQueryExecutionCommand,\n} from '@aws-sdk/client-athena';\n\n/**\n * Executes the given command and waits up to 10s for the command to complete.\n * @param failOnTimeout <code>true</code> in order to throw an exception,\n * or <code>false</code> to just log the error to the console\n * @return <code>true</code> on success or <code>false</code> if the command timed out and\n * <code>failOnTimeout</code> was set to false\n */\nexport const executeAndAwaitResponse = async (\n  command: StartQueryExecutionCommand,\n  failOnTimeout: boolean\n): Promise<boolean> => {\n  console.log('executing command ', command.input.QueryString, '...');\n\n  const athena = new AthenaClient({});\n  const response = await athena.send(command);\n\n  // wait up to 10s for completion\n  const checkStatus = new GetQueryExecutionCommand({\n    QueryExecutionId: response.QueryExecutionId,\n  });\n\n  for (let i = 1; i <= 50; i++) {\n    const state = await athena.send(checkStatus);\n    switch (state.QueryExecution!.Status!.State) {\n      case QueryExecutionState.SUCCEEDED:\n        console.log('command completed successfully');\n        return true;\n      case QueryExecutionState.CANCELLED:\n      case QueryExecutionState.FAILED:\n        // noinspection TypeScriptUnresolvedFunction\n        throw new Error('Command execution failed! Query:\\n' + command.input.QueryString);\n    }\n\n    await new Promise(resolve => setTimeout(resolve, 200));\n  }\n\n  const errorMessage = 'Status of partition creation unknown - waited for 10s but command did not complete';\n  if (failOnTimeout) {\n    // noinspection TypeScriptUnresolvedFunction\n    throw new Error(errorMessage);\n  } else {\n    console.error(errorMessage);\n  }\n  return false;\n};\n"]}

package/lib/functions/access-logs-analysis/partitioning/create-partition.d.ts

@@ -0,0 +1 @@
+export declare const handler: () => Promise<void>;

package/lib/functions/access-logs-analysis/partitioning/create-partition.js

@@ -0,0 +1,57 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handler = void 0;
+/**
+ * This script creates a partition for the upcoming hour.
+ * Taken and adjusted from https://github.com/aws-samples/amazon-cloudfront-access-logs-queries/blob/mainline/functions/createPartitions.js.
+ */
+// noinspection DuplicatedCode
+const client_athena_1 = require("@aws-sdk/client-athena");
+const util_1 = require("./util");
+const workgroup = process.env.WORKGROUP;
+if (!workgroup) {
+    throw new Error('Required environment variable WORKGROUP missing!');
+}
+const database = process.env.DATABASE;
+if (!database) {
+    throw new Error('Required environment variable DATABASE missing!');
+}
+const table = process.env.TABLE;
+if (!table) {
+    throw new Error('Required environment variable TABLE missing!');
+}
+const internalHandler = async () => {
+    try {
+        const nextHour = new Date(Date.now() + 60 * 60 * 1000);
+        const year = nextHour.getUTCFullYear();
+        const month = (nextHour.getUTCMonth() + 1).toString().padStart(2, '0');
+        const day = nextHour
+            .getUTCDate()
+            .toString()
+            .padStart(2, '0');
+        const hour = nextHour
+            .getUTCHours()
+            .toString()
+            .padStart(2, '0');
+        console.log('creating partition', { year, month, day, hour });
+        const command = new client_athena_1.StartQueryExecutionCommand({
+            QueryString: `
+        ALTER TABLE ${database}.${table}
+        ADD IF NOT EXISTS
+        PARTITION (
+            year = '${year}',
+            month = '${month}',
+            day = '${day}',
+            hour = '${hour}' );`,
+            WorkGroup: workgroup,
+            QueryExecutionContext: { Database: database },
+        });
+        await (0, util_1.executeAndAwaitResponse)(command, true);
+        console.log('partition successfully created');
+    }
+    catch (error) {
+        console.error('### unexpected runtime error ###', error);
+    }
+};
+exports.handler = internalHandler;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY3JlYXRlLXBhcnRpdGlvbi5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbImNyZWF0ZS1wYXJ0aXRpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7O0FBQUE7OztHQUdHO0FBQ0gsOEJBQThCO0FBQzlCLDBEQUFrRTtBQUNsRSxpQ0FBK0M7QUFFL0MsTUFBTSxTQUFTLEdBQUcsT0FBTyxDQUFDLEdBQUcsQ0FBQyxTQUFTLENBQUM7QUFDeEMsSUFBSSxDQUFDLFNBQVMsRUFBRTtJQUNkLE1BQU0sSUFBSSxLQUFLLENBQUMsa0RBQWtELENBQUMsQ0FBQztDQUNyRTtBQUNELE1BQU0sUUFBUSxHQUFHLE9BQU8sQ0FBQyxHQUFHLENBQUMsUUFBUSxDQUFDO0FBQ3RDLElBQUksQ0FBQyxRQUFRLEVBQUU7SUFDYixNQUFNLElBQUksS0FBSyxDQUFDLGlEQUFpRCxDQUFDLENBQUM7Q0FDcEU7QUFDRCxNQUFNLEtBQUssR0FBRyxPQUFPLENBQUMsR0FBRyxDQUFDLEtBQUssQ0FBQztBQUNoQyxJQUFJLENBQUMsS0FBSyxFQUFFO0lBQ1YsTUFBTSxJQUFJLEtBQUssQ0FBQyw4Q0FBOEMsQ0FBQyxDQUFDO0NBQ2pFO0FBRUQsTUFBTSxlQUFlLEdBQUcsS0FBSyxJQUFJLEVBQUU7SUFDakMsSUFBSTtRQUNGLE1BQU0sUUFBUSxHQUFHLElBQUksSUFBSSxDQUFDLElBQUksQ0FBQyxHQUFHLEVBQUUsR0FBRyxFQUFFLEdBQUcsRUFBRSxHQUFHLElBQUksQ0FBQyxDQUFDO1FBQ3ZELE1BQU0sSUFBSSxHQUFHLFFBQVEsQ0FBQyxjQUFjLEVBQUUsQ0FBQztRQUN2QyxNQUFNLEtBQUssR0FBRyxDQUFDLFFBQVEsQ0FBQyxXQUFXLEVBQUUsR0FBRyxDQUFDLENBQUMsQ0FBQyxRQUFRLEVBQUUsQ0FBQyxRQUFRLENBQUMsQ0FBQyxFQUFFLEdBQUcsQ0FBQyxDQUFDO1FBQ3ZFLE1BQU0sR0FBRyxHQUFHLFFBQVE7YUFDakIsVUFBVSxFQUFFO2FBQ1osUUFBUSxFQUFFO2FBQ1YsUUFBUSxDQUFDLENBQUMsRUFBRSxHQUFHLENBQUMsQ0FBQztRQUNwQixNQUFNLElBQUksR0FBRyxRQUFRO2FBQ2xCLFdBQVcsRUFBRTthQUNiLFFBQVEsRUFBRTthQUNWLFFBQVEsQ0FBQyxDQUFDLEVBQUUsR0FBRyxDQUFDLENBQUM7UUFFcEIsT0FBTyxDQUFDLEdBQUcsQ0FBQyxvQkFBb0IsRUFBRSxFQUFDLElBQUksRUFBRSxLQUFLLEVBQUUsR0FBRyxFQUFFLElBQUksRUFBQyxDQUFDLENBQUM7UUFFNUQsTUFBTSxPQUFPLEdBQUcsSUFBSSwwQ0FBMEIsQ0FBQztZQUM3QyxXQUFXLEVBQUU7c0JBQ0csUUFBUSxJQUFJLEtBQUs7OztzQkFHakIsSUFBSTt1QkFDSCxLQUFLO3FCQUNQLEdBQUc7c0JBQ0YsSUFBSSxNQUFNO1lBQzFCLFNBQVMsRUFBRSxTQUFTO1lBQ3BCLHFCQUFxQixFQUFFLEVBQUMsUUFBUSxFQUFFLFFBQVEsRUFBQztTQUM1QyxDQUFDLENBQUM7UUFFSCxNQUFNLElBQUEsOEJBQXVCLEVBQUMsT0FBTyxFQUFFLElBQUksQ0FBQyxDQUFDO1FBQzdDLE9BQU8sQ0FBQyxHQUFHLENBQUMsZ0NBQWdDLENBQUMsQ0FBQztLQUMvQztJQUFDLE9BQU8sS0FBSyxFQUFFO1FBQ2QsT0FBTyxDQUFDLEtBQUssQ0FBQyxrQ0FBa0MsRUFBRSxLQUFLLENBQUMsQ0FBQztLQUMxRDtBQUNILENBQUMsQ0FBQztBQUVXLFFBQUEsT0FBTyxHQUFHLGVBQWUsQ0FBQyIsInNvdXJjZXNDb250ZW50IjpbIi8qKlxuICogVGhpcyBzY3JpcHQgY3JlYXRlcyBhIHBhcnRpdGlvbiBmb3IgdGhlIHVwY29taW5nIGhvdXIuXG4gKiBUYWtlbiBhbmQgYWRqdXN0ZWQgZnJvbSBodHRwczovL2dpdGh1Yi5jb20vYXdzLXNhbXBsZXMvYW1hem9uLWNsb3VkZnJvbnQtYWNjZXNzLWxvZ3MtcXVlcmllcy9ibG9iL21haW5saW5lL2Z1bmN0aW9ucy9jcmVhdGVQYXJ0aXRpb25zLmpzLlxuICovXG4vLyBub2luc3BlY3Rpb24gRHVwbGljYXRlZENvZGVcbmltcG9ydCB7U3RhcnRRdWVyeUV4ZWN1dGlvbkNvbW1hbmR9IGZyb20gJ0Bhd3Mtc2RrL2NsaWVudC1hdGhlbmEnO1xuaW1wb3J0IHtleGVjdXRlQW5kQXdhaXRSZXNwb25zZX0gZnJvbSAnLi91dGlsJztcblxuY29uc3Qgd29ya2dyb3VwID0gcHJvY2Vzcy5lbnYuV09SS0dST1VQO1xuaWYgKCF3b3JrZ3JvdXApIHtcbiAgdGhyb3cgbmV3IEVycm9yKCdSZXF1aXJlZCBlbnZpcm9ubWVudCB2YXJpYWJsZSBXT1JLR1JPVVAgbWlzc2luZyEnKTtcbn1cbmNvbnN0IGRhdGFiYXNlID0gcHJvY2Vzcy5lbnYuREFUQUJBU0U7XG5pZiAoIWRhdGFiYXNlKSB7XG4gIHRocm93IG5ldyBFcnJvcignUmVxdWlyZWQgZW52aXJvbm1lbnQgdmFyaWFibGUgREFUQUJBU0UgbWlzc2luZyEnKTtcbn1cbmNvbnN0IHRhYmxlID0gcHJvY2Vzcy5lbnYuVEFCTEU7XG5pZiAoIXRhYmxlKSB7XG4gIHRocm93IG5ldyBFcnJvcignUmVxdWlyZWQgZW52aXJvbm1lbnQgdmFyaWFibGUgVEFCTEUgbWlzc2luZyEnKTtcbn1cblxuY29uc3QgaW50ZXJuYWxIYW5kbGVyID0gYXN5bmMgKCkgPT4ge1xuICB0cnkge1xuICAgIGNvbnN0IG5leHRIb3VyID0gbmV3IERhdGUoRGF0ZS5ub3coKSArIDYwICogNjAgKiAxMDAwKTtcbiAgICBjb25zdCB5ZWFyID0gbmV4dEhvdXIuZ2V0VVRDRnVsbFllYXIoKTtcbiAgICBjb25zdCBtb250aCA9IChuZXh0SG91ci5nZXRVVENNb250aCgpICsgMSkudG9TdHJpbmcoKS5wYWRTdGFydCgyLCAnMCcpO1xuICAgIGNvbnN0IGRheSA9IG5leHRIb3VyXG4gICAgICAuZ2V0VVRDRGF0ZSgpXG4gICAgICAudG9TdHJpbmcoKVxuICAgICAgLnBhZFN0YXJ0KDIsICcwJyk7XG4gICAgY29uc3QgaG91ciA9IG5leHRIb3VyXG4gICAgICAuZ2V0VVRDSG91cnMoKVxuICAgICAgLnRvU3RyaW5nKClcbiAgICAgIC5wYWRTdGFydCgyLCAnMCcpO1xuXG4gICAgY29uc29sZS5sb2coJ2NyZWF0aW5nIHBhcnRpdGlvbicsIHt5ZWFyLCBtb250aCwgZGF5LCBob3VyfSk7XG5cbiAgICBjb25zdCBjb21tYW5kID0gbmV3IFN0YXJ0UXVlcnlFeGVjdXRpb25Db21tYW5kKHtcbiAgICAgIFF1ZXJ5U3RyaW5nOiBgXG4gICAgICAgIEFMVEVSIFRBQkxFICR7ZGF0YWJhc2V9LiR7dGFibGV9XG4gICAgICAgIEFERCBJRiBOT1QgRVhJU1RTXG4gICAgICAgIFBBUlRJVElPTiAoXG4gICAgICAgICAgICB5ZWFyID0gJyR7eWVhcn0nLFxuICAgICAgICAgICAgbW9udGggPSAnJHttb250aH0nLFxuICAgICAgICAgICAgZGF5ID0gJyR7ZGF5fScsXG4gICAgICAgICAgICBob3VyID0gJyR7aG91cn0nICk7YCxcbiAgICAgIFdvcmtHcm91cDogd29ya2dyb3VwLFxuICAgICAgUXVlcnlFeGVjdXRpb25Db250ZXh0OiB7RGF0YWJhc2U6IGRhdGFiYXNlfSxcbiAgICB9KTtcblxuICAgIGF3YWl0IGV4ZWN1dGVBbmRBd2FpdFJlc3BvbnNlKGNvbW1hbmQsIHRydWUpO1xuICAgIGNvbnNvbGUubG9nKCdwYXJ0aXRpb24gc3VjY2Vzc2Z1bGx5IGNyZWF0ZWQnKTtcbiAgfSBjYXRjaCAoZXJyb3IpIHtcbiAgICBjb25zb2xlLmVycm9yKCcjIyMgdW5leHBlY3RlZCBydW50aW1lIGVycm9yICMjIycsIGVycm9yKTtcbiAgfVxufTtcblxuZXhwb3J0IGNvbnN0IGhhbmRsZXIgPSBpbnRlcm5hbEhhbmRsZXI7XG4iXX0=

package/lib/functions/access-logs-analysis/partitioning/create-partition.ts

@@ -0,0 +1,58 @@
+/**
+ * This script creates a partition for the upcoming hour.
+ * Taken and adjusted from https://github.com/aws-samples/amazon-cloudfront-access-logs-queries/blob/mainline/functions/createPartitions.js.
+ */
+// noinspection DuplicatedCode
+import {StartQueryExecutionCommand} from '@aws-sdk/client-athena';
+import {executeAndAwaitResponse} from './util';
+
+const workgroup = process.env.WORKGROUP;
+if (!workgroup) {
+  throw new Error('Required environment variable WORKGROUP missing!');
+}
+const database = process.env.DATABASE;
+if (!database) {
+  throw new Error('Required environment variable DATABASE missing!');
+}
+const table = process.env.TABLE;
+if (!table) {
+  throw new Error('Required environment variable TABLE missing!');
+}
+
+const internalHandler = async () => {
+  try {
+    const nextHour = new Date(Date.now() + 60 * 60 * 1000);
+    const year = nextHour.getUTCFullYear();
+    const month = (nextHour.getUTCMonth() + 1).toString().padStart(2, '0');
+    const day = nextHour
+      .getUTCDate()
+      .toString()
+      .padStart(2, '0');
+    const hour = nextHour
+      .getUTCHours()
+      .toString()
+      .padStart(2, '0');
+
+    console.log('creating partition', {year, month, day, hour});
+
+    const command = new StartQueryExecutionCommand({
+      QueryString: `
+        ALTER TABLE ${database}.${table}
+        ADD IF NOT EXISTS
+        PARTITION (
+            year = '${year}',
+            month = '${month}',
+            day = '${day}',
+            hour = '${hour}' );`,
+      WorkGroup: workgroup,
+      QueryExecutionContext: {Database: database},
+    });
+
+    await executeAndAwaitResponse(command, true);
+    console.log('partition successfully created');
+  } catch (error) {
+    console.error('### unexpected runtime error ###', error);
+  }
+};
+
+export const handler = internalHandler;

package/lib/functions/access-logs-analysis/partitioning/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "cdk-nuxt-access-log-analysis-create-partitions",
+  "version": "1.0.0",
+  "main": "index.js",
+  "license": "UNLICENSED",
+  "private": true,
+  "scripts": {
+    "install-layer": "yarn install --frozen-lockfile --production --modules-folder build/layer/nodejs/node_modules",
+    "build": "tsc --project ./tsconfig.json"
+  },
+  "devDependencies": {
+    "@types/node": "^16.11.0",
+    "@types/aws-lambda": "^8.10.101",
+    "ts-node": "^10.9.1",
+    "typescript": "^4.7.4"
+  },
+  "dependencies": {
+    "@aws-sdk/client-athena": "^3.131.0"
+  }
+}

package/lib/functions/access-logs-analysis/partitioning/transform-partition.d.ts

@@ -0,0 +1,2 @@
+import type { TransformPartitionEvent } from './types';
+export declare const handler: (event: TransformPartitionEvent) => Promise<void>;

package/lib/functions/access-logs-analysis/partitioning/transform-partition.js

@@ -0,0 +1,72 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handler = void 0;
+/**
+ * This script transforms the access log partition of two hours ago into the Apache parquet format.
+ * Taken and adjusted from https://github.com/aws-samples/amazon-cloudfront-access-logs-queries/blob/mainline/functions/transformPartition.js.
+ */
+const client_athena_1 = require("@aws-sdk/client-athena");
+const util_1 = require("./util");
+const workgroup = process.env.WORKGROUP;
+if (!workgroup) {
+    throw new Error('Required environment variable WORKGROUP missing!');
+}
+const database = process.env.DATABASE;
+if (!database) {
+    throw new Error('Required environment variable DATABASE missing!');
+}
+const sourceTable = process.env.SOURCE_TABLE;
+if (!sourceTable) {
+    throw new Error('Required environment variable SOURCE_TABLE missing!');
+}
+const targetTable = process.env.TARGET_TABLE;
+if (!targetTable) {
+    throw new Error('Required environment variable TARGET_TABLE missing!');
+}
+/**
+ * Reducer, that computes the column statements for the insert command. Columns are either selected by their name, or
+ * are computed by an expression specified in `columnTransformations`.
+ */
+const buildQueryColumns = (columnTransformations, previous, current) => {
+    const columnExpression = Object.keys(columnTransformations).includes(current)
+        ? `${columnTransformations[current]} AS ${current}`
+        : current;
+    return previous.length === 0 ? columnExpression : `${previous}, ${columnExpression}`;
+};
+/**
+ * Entrypoint
+ */
+const internalHandler = async (event) => {
+    try {
+        const partitionHour = new Date(Date.now() - 120 * 60 * 1000);
+        const year = partitionHour.getUTCFullYear();
+        const month = (partitionHour.getUTCMonth() + 1).toString().padStart(2, '0');
+        const day = partitionHour.getUTCDate().toString().padStart(2, '0');
+        const hour = partitionHour.getUTCHours().toString().padStart(2, '0');
+        console.log('transforming partition', { year, month, day, hour });
+        // apply transformations on certain columns:
+        // - obfuscate IPs
+        // - optionally filter cookies
+        const selectExpr = event.columnNames.reduce((previous, current) => buildQueryColumns(event.columnTransformations, previous, current), '');
+        const command = new client_athena_1.StartQueryExecutionCommand({
+            QueryString: `
+        INSERT INTO ${database}.${targetTable} (${event.columnNames.join(',')})
+        SELECT ${selectExpr}
+        FROM ${database}.${sourceTable}
+        WHERE year = '${year}'
+          AND month = '${month}'
+          AND day = '${day}'
+          AND hour = '${hour}';`,
+            WorkGroup: workgroup,
+            QueryExecutionContext: { Database: database },
+        });
+        if (await (0, util_1.executeAndAwaitResponse)(command, false)) {
+            console.log('successfully transformed partition', { year, month, day, hour });
+        }
+    }
+    catch (error) {
+        console.error('### unexpected runtime error ###', error);
+    }
+};
+exports.handler = internalHandler;
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidHJhbnNmb3JtLXBhcnRpdGlvbi5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbInRyYW5zZm9ybS1wYXJ0aXRpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7O0FBQUE7OztHQUdHO0FBQ0gsMERBQWtFO0FBQ2xFLGlDQUErQztBQUcvQyxNQUFNLFNBQVMsR0FBRyxPQUFPLENBQUMsR0FBRyxDQUFDLFNBQVMsQ0FBQztBQUN4QyxJQUFJLENBQUMsU0FBUyxFQUFFO0lBQ2QsTUFBTSxJQUFJLEtBQUssQ0FBQyxrREFBa0QsQ0FBQyxDQUFDO0NBQ3JFO0FBQ0QsTUFBTSxRQUFRLEdBQUcsT0FBTyxDQUFDLEdBQUcsQ0FBQyxRQUFRLENBQUM7QUFDdEMsSUFBSSxDQUFDLFFBQVEsRUFBRTtJQUNiLE1BQU0sSUFBSSxLQUFLLENBQUMsaURBQWlELENBQUMsQ0FBQztDQUNwRTtBQUNELE1BQU0sV0FBVyxHQUFHLE9BQU8sQ0FBQyxHQUFHLENBQUMsWUFBWSxDQUFDO0FBQzdDLElBQUksQ0FBQyxXQUFXLEVBQUU7SUFDaEIsTUFBTSxJQUFJLEtBQUssQ0FBQyxxREFBcUQsQ0FBQyxDQUFDO0NBQ3hFO0FBQ0QsTUFBTSxXQUFXLEdBQUcsT0FBTyxDQUFDLEdBQUcsQ0FBQyxZQUFZLENBQUM7QUFDN0MsSUFBSSxDQUFDLFdBQVcsRUFBRTtJQUNoQixNQUFNLElBQUksS0FBSyxDQUFDLHFEQUFxRCxDQUFDLENBQUM7Q0FDeEU7QUFFRDs7O0dBR0c7QUFDSCxNQUFNLGlCQUFpQixHQUFHLENBQ3hCLHFCQUFnRCxFQUNoRCxRQUFnQixFQUNoQixPQUFlLEVBQ1AsRUFBRTtJQUNWLE1BQU0sZ0JBQWdCLEdBQUcsTUFBTSxDQUFDLElBQUksQ0FBQyxxQkFBcUIsQ0FBQyxDQUFDLFFBQVEsQ0FBQyxPQUFPLENBQUM7UUFDM0UsQ0FBQyxDQUFDLEdBQUcscUJBQXFCLENBQUMsT0FBTyxDQUFDLE9BQU8sT0FBTyxFQUFFO1FBQ25ELENBQUMsQ0FBQyxPQUFPLENBQUM7SUFDWixPQUFPLFFBQVEsQ0FBQyxNQUFNLEtBQUssQ0FBQyxDQUFDLENBQUMsQ0FBQyxnQkFBZ0IsQ0FBQyxDQUFDLENBQUMsR0FBRyxRQUFRLEtBQUssZ0JBQWdCLEVBQUUsQ0FBQztBQUN2RixDQUFDLENBQUM7QUFFRjs7R0FFRztBQUNILE1BQU0sZUFBZSxHQUFHLEtBQUssRUFBRSxLQUE4QixFQUFFLEVBQUU7SUFDL0QsSUFBSTtRQUNGLE1BQU0sYUFBYSxHQUFHLElBQUksSUFBSSxDQUFDLElBQUksQ0FBQyxHQUFHLEVBQUUsR0FBRyxHQUFHLEdBQUcsRUFBRSxHQUFHLElBQUksQ0FBQyxDQUFDO1FBQzdELE1BQU0sSUFBSSxHQUFHLGFBQWEsQ0FBQyxjQUFjLEVBQUUsQ0FBQztRQUM1QyxNQUFNLEtBQUssR0FBRyxDQUFDLGFBQWEsQ0FBQyxXQUFXLEVBQUUsR0FBRyxDQUFDLENBQUMsQ0FBQyxRQUFRLEVBQUUsQ0FBQyxRQUFRLENBQUMsQ0FBQyxFQUFFLEdBQUcsQ0FBQyxDQUFDO1FBQzVFLE1BQU0sR0FBRyxHQUFHLGFBQWEsQ0FBQyxVQUFVLEVBQUUsQ0FBQyxRQUFRLEVBQUUsQ0FBQyxRQUFRLENBQUMsQ0FBQyxFQUFFLEdBQUcsQ0FBQyxDQUFDO1FBQ25FLE1BQU0sSUFBSSxHQUFHLGFBQWEsQ0FBQyxXQUFXLEVBQUUsQ0FBQyxRQUFRLEVBQUUsQ0FBQyxRQUFRLENBQUMsQ0FBQyxFQUFFLEdBQUcsQ0FBQyxDQUFDO1FBRXJFLE9BQU8sQ0FBQyxHQUFHLENBQUMsd0JBQXdCLEVBQUUsRUFBQyxJQUFJLEVBQUUsS0FBSyxFQUFFLEdBQUcsRUFBRSxJQUFJLEVBQUMsQ0FBQyxDQUFDO1FBRWhFLDRDQUE0QztRQUM1QyxrQkFBa0I7UUFDbEIsOEJBQThCO1FBQzlCLE1BQU0sVUFBVSxHQUFHLEtBQUssQ0FBQyxXQUFXLENBQUMsTUFBTSxDQUN6QyxDQUFDLFFBQVEsRUFBRSxPQUFPLEVBQUUsRUFBRSxDQUFDLGlCQUFpQixDQUFDLEtBQUssQ0FBQyxxQkFBcUIsRUFBRSxRQUFRLEVBQUUsT0FBTyxDQUFDLEVBQ3hGLEVBQUUsQ0FDSCxDQUFDO1FBRUYsTUFBTSxPQUFPLEdBQUcsSUFBSSwwQ0FBMEIsQ0FBQztZQUM3QyxXQUFXLEVBQUU7c0JBQ0csUUFBUSxJQUFJLFdBQVcsS0FBSyxLQUFLLENBQUMsV0FBVyxDQUFDLElBQUksQ0FBQyxHQUFHLENBQUM7aUJBQzVELFVBQVU7ZUFDWixRQUFRLElBQUksV0FBVzt3QkFDZCxJQUFJO3lCQUNILEtBQUs7dUJBQ1AsR0FBRzt3QkFDRixJQUFJLElBQUk7WUFDMUIsU0FBUyxFQUFFLFNBQVM7WUFDcEIscUJBQXFCLEVBQUUsRUFBQyxRQUFRLEVBQUUsUUFBUSxFQUFDO1NBQzVDLENBQUMsQ0FBQztRQUVILElBQUksTUFBTSxJQUFBLDhCQUF1QixFQUFDLE9BQU8sRUFBRSxLQUFLLENBQUMsRUFBRTtZQUNqRCxPQUFPLENBQUMsR0FBRyxDQUFDLG9DQUFvQyxFQUFFLEVBQUMsSUFBSSxFQUFFLEtBQUssRUFBRSxHQUFHLEVBQUUsSUFBSSxFQUFDLENBQUMsQ0FBQztTQUM3RTtLQUNGO0lBQUMsT0FBTyxLQUFLLEVBQUU7UUFDZCxPQUFPLENBQUMsS0FBSyxDQUFDLGtDQUFrQyxFQUFFLEtBQUssQ0FBQyxDQUFDO0tBQzFEO0FBQ0gsQ0FBQyxDQUFDO0FBRVcsUUFBQSxPQUFPLEdBQUcsZUFBZSxDQUFDIiwic291cmNlc0NvbnRlbnQiOlsiLyoqXG4gKiBUaGlzIHNjcmlwdCB0cmFuc2Zvcm1zIHRoZSBhY2Nlc3MgbG9nIHBhcnRpdGlvbiBvZiB0d28gaG91cnMgYWdvIGludG8gdGhlIEFwYWNoZSBwYXJxdWV0IGZvcm1hdC5cbiAqIFRha2VuIGFuZCBhZGp1c3RlZCBmcm9tIGh0dHBzOi8vZ2l0aHViLmNvbS9hd3Mtc2FtcGxlcy9hbWF6b24tY2xvdWRmcm9udC1hY2Nlc3MtbG9ncy1xdWVyaWVzL2Jsb2IvbWFpbmxpbmUvZnVuY3Rpb25zL3RyYW5zZm9ybVBhcnRpdGlvbi5qcy5cbiAqL1xuaW1wb3J0IHtTdGFydFF1ZXJ5RXhlY3V0aW9uQ29tbWFuZH0gZnJvbSAnQGF3cy1zZGsvY2xpZW50LWF0aGVuYSc7XG5pbXBvcnQge2V4ZWN1dGVBbmRBd2FpdFJlc3BvbnNlfSBmcm9tICcuL3V0aWwnO1xuaW1wb3J0IHR5cGUge0NvbHVtblRyYW5zZm9ybWF0aW9uUnVsZXMsIFRyYW5zZm9ybVBhcnRpdGlvbkV2ZW50fSBmcm9tICcuL3R5cGVzJztcblxuY29uc3Qgd29ya2dyb3VwID0gcHJvY2Vzcy5lbnYuV09SS0dST1VQO1xuaWYgKCF3b3JrZ3JvdXApIHtcbiAgdGhyb3cgbmV3IEVycm9yKCdSZXF1aXJlZCBlbnZpcm9ubWVudCB2YXJpYWJsZSBXT1JLR1JPVVAgbWlzc2luZyEnKTtcbn1cbmNvbnN0IGRhdGFiYXNlID0gcHJvY2Vzcy5lbnYuREFUQUJBU0U7XG5pZiAoIWRhdGFiYXNlKSB7XG4gIHRocm93IG5ldyBFcnJvcignUmVxdWlyZWQgZW52aXJvbm1lbnQgdmFyaWFibGUgREFUQUJBU0UgbWlzc2luZyEnKTtcbn1cbmNvbnN0IHNvdXJjZVRhYmxlID0gcHJvY2Vzcy5lbnYuU09VUkNFX1RBQkxFO1xuaWYgKCFzb3VyY2VUYWJsZSkge1xuICB0aHJvdyBuZXcgRXJyb3IoJ1JlcXVpcmVkIGVudmlyb25tZW50IHZhcmlhYmxlIFNPVVJDRV9UQUJMRSBtaXNzaW5nIScpO1xufVxuY29uc3QgdGFyZ2V0VGFibGUgPSBwcm9jZXNzLmVudi5UQVJHRVRfVEFCTEU7XG5pZiAoIXRhcmdldFRhYmxlKSB7XG4gIHRocm93IG5ldyBFcnJvcignUmVxdWlyZWQgZW52aXJvbm1lbnQgdmFyaWFibGUgVEFSR0VUX1RBQkxFIG1pc3NpbmchJyk7XG59XG5cbi8qKlxuICogUmVkdWNlciwgdGhhdCBjb21wdXRlcyB0aGUgY29sdW1uIHN0YXRlbWVudHMgZm9yIHRoZSBpbnNlcnQgY29tbWFuZC4gQ29sdW1ucyBhcmUgZWl0aGVyIHNlbGVjdGVkIGJ5IHRoZWlyIG5hbWUsIG9yXG4gKiBhcmUgY29tcHV0ZWQgYnkgYW4gZXhwcmVzc2lvbiBzcGVjaWZpZWQgaW4gYGNvbHVtblRyYW5zZm9ybWF0aW9uc2AuXG4gKi9cbmNvbnN0IGJ1aWxkUXVlcnlDb2x1bW5zID0gKFxuICBjb2x1bW5UcmFuc2Zvcm1hdGlvbnM6IENvbHVtblRyYW5zZm9ybWF0aW9uUnVsZXMsXG4gIHByZXZpb3VzOiBzdHJpbmcsXG4gIGN1cnJlbnQ6IHN0cmluZ1xuKTogc3RyaW5nID0+IHtcbiAgY29uc3QgY29sdW1uRXhwcmVzc2lvbiA9IE9iamVjdC5rZXlzKGNvbHVtblRyYW5zZm9ybWF0aW9ucykuaW5jbHVkZXMoY3VycmVudClcbiAgICA/IGAke2NvbHVtblRyYW5zZm9ybWF0aW9uc1tjdXJyZW50XX0gQVMgJHtjdXJyZW50fWBcbiAgICA6IGN1cnJlbnQ7XG4gIHJldHVybiBwcmV2aW91cy5sZW5ndGggPT09IDAgPyBjb2x1bW5FeHByZXNzaW9uIDogYCR7cHJldmlvdXN9LCAke2NvbHVtbkV4cHJlc3Npb259YDtcbn07XG5cbi8qKlxuICogRW50cnlwb2ludFxuICovXG5jb25zdCBpbnRlcm5hbEhhbmRsZXIgPSBhc3luYyAoZXZlbnQ6IFRyYW5zZm9ybVBhcnRpdGlvbkV2ZW50KSA9PiB7XG4gIHRyeSB7XG4gICAgY29uc3QgcGFydGl0aW9uSG91ciA9IG5ldyBEYXRlKERhdGUubm93KCkgLSAxMjAgKiA2MCAqIDEwMDApO1xuICAgIGNvbnN0IHllYXIgPSBwYXJ0aXRpb25Ib3VyLmdldFVUQ0Z1bGxZZWFyKCk7XG4gICAgY29uc3QgbW9udGggPSAocGFydGl0aW9uSG91ci5nZXRVVENNb250aCgpICsgMSkudG9TdHJpbmcoKS5wYWRTdGFydCgyLCAnMCcpO1xuICAgIGNvbnN0IGRheSA9IHBhcnRpdGlvbkhvdXIuZ2V0VVRDRGF0ZSgpLnRvU3RyaW5nKCkucGFkU3RhcnQoMiwgJzAnKTtcbiAgICBjb25zdCBob3VyID0gcGFydGl0aW9uSG91ci5nZXRVVENIb3VycygpLnRvU3RyaW5nKCkucGFkU3RhcnQoMiwgJzAnKTtcblxuICAgIGNvbnNvbGUubG9nKCd0cmFuc2Zvcm1pbmcgcGFydGl0aW9uJywge3llYXIsIG1vbnRoLCBkYXksIGhvdXJ9KTtcblxuICAgIC8vIGFwcGx5IHRyYW5zZm9ybWF0aW9ucyBvbiBjZXJ0YWluIGNvbHVtbnM6XG4gICAgLy8gLSBvYmZ1c2NhdGUgSVBzXG4gICAgLy8gLSBvcHRpb25hbGx5IGZpbHRlciBjb29raWVzXG4gICAgY29uc3Qgc2VsZWN0RXhwciA9IGV2ZW50LmNvbHVtbk5hbWVzLnJlZHVjZShcbiAgICAgIChwcmV2aW91cywgY3VycmVudCkgPT4gYnVpbGRRdWVyeUNvbHVtbnMoZXZlbnQuY29sdW1uVHJhbnNmb3JtYXRpb25zLCBwcmV2aW91cywgY3VycmVudCksXG4gICAgICAnJ1xuICAgICk7XG5cbiAgICBjb25zdCBjb21tYW5kID0gbmV3IFN0YXJ0UXVlcnlFeGVjdXRpb25Db21tYW5kKHtcbiAgICAgIFF1ZXJ5U3RyaW5nOiBgXG4gICAgICAgIElOU0VSVCBJTlRPICR7ZGF0YWJhc2V9LiR7dGFyZ2V0VGFibGV9ICgke2V2ZW50LmNvbHVtbk5hbWVzLmpvaW4oJywnKX0pXG4gICAgICAgIFNFTEVDVCAke3NlbGVjdEV4cHJ9XG4gICAgICAgIEZST00gJHtkYXRhYmFzZX0uJHtzb3VyY2VUYWJsZX1cbiAgICAgICAgV0hFUkUgeWVhciA9ICcke3llYXJ9J1xuICAgICAgICAgIEFORCBtb250aCA9ICcke21vbnRofSdcbiAgICAgICAgICBBTkQgZGF5ID0gJyR7ZGF5fSdcbiAgICAgICAgICBBTkQgaG91ciA9ICcke2hvdXJ9JztgLFxuICAgICAgV29ya0dyb3VwOiB3b3JrZ3JvdXAsXG4gICAgICBRdWVyeUV4ZWN1dGlvbkNvbnRleHQ6IHtEYXRhYmFzZTogZGF0YWJhc2V9LFxuICAgIH0pO1xuXG4gICAgaWYgKGF3YWl0IGV4ZWN1dGVBbmRBd2FpdFJlc3BvbnNlKGNvbW1hbmQsIGZhbHNlKSkge1xuICAgICAgY29uc29sZS5sb2coJ3N1Y2Nlc3NmdWxseSB0cmFuc2Zvcm1lZCBwYXJ0aXRpb24nLCB7eWVhciwgbW9udGgsIGRheSwgaG91cn0pO1xuICAgIH1cbiAgfSBjYXRjaCAoZXJyb3IpIHtcbiAgICBjb25zb2xlLmVycm9yKCcjIyMgdW5leHBlY3RlZCBydW50aW1lIGVycm9yICMjIycsIGVycm9yKTtcbiAgfVxufTtcblxuZXhwb3J0IGNvbnN0IGhhbmRsZXIgPSBpbnRlcm5hbEhhbmRsZXI7XG4iXX0=

package/lib/functions/access-logs-analysis/partitioning/transform-partition.ts

@@ -0,0 +1,83 @@
+/**
+ * This script transforms the access log partition of two hours ago into the Apache parquet format.
+ * Taken and adjusted from https://github.com/aws-samples/amazon-cloudfront-access-logs-queries/blob/mainline/functions/transformPartition.js.
+ */
+import {StartQueryExecutionCommand} from '@aws-sdk/client-athena';
+import {executeAndAwaitResponse} from './util';
+import type {ColumnTransformationRules, TransformPartitionEvent} from './types';
+
+const workgroup = process.env.WORKGROUP;
+if (!workgroup) {
+  throw new Error('Required environment variable WORKGROUP missing!');
+}
+const database = process.env.DATABASE;
+if (!database) {
+  throw new Error('Required environment variable DATABASE missing!');
+}
+const sourceTable = process.env.SOURCE_TABLE;
+if (!sourceTable) {
+  throw new Error('Required environment variable SOURCE_TABLE missing!');
+}
+const targetTable = process.env.TARGET_TABLE;
+if (!targetTable) {
+  throw new Error('Required environment variable TARGET_TABLE missing!');
+}
+
+/**
+ * Reducer, that computes the column statements for the insert command. Columns are either selected by their name, or
+ * are computed by an expression specified in `columnTransformations`.
+ */
+const buildQueryColumns = (
+  columnTransformations: ColumnTransformationRules,
+  previous: string,
+  current: string
+): string => {
+  const columnExpression = Object.keys(columnTransformations).includes(current)
+    ? `${columnTransformations[current]} AS ${current}`
+    : current;
+  return previous.length === 0 ? columnExpression : `${previous}, ${columnExpression}`;
+};
+
+/**
+ * Entrypoint
+ */
+const internalHandler = async (event: TransformPartitionEvent) => {
+  try {
+    const partitionHour = new Date(Date.now() - 120 * 60 * 1000);
+    const year = partitionHour.getUTCFullYear();
+    const month = (partitionHour.getUTCMonth() + 1).toString().padStart(2, '0');
+    const day = partitionHour.getUTCDate().toString().padStart(2, '0');
+    const hour = partitionHour.getUTCHours().toString().padStart(2, '0');
+
+    console.log('transforming partition', {year, month, day, hour});
+
+    // apply transformations on certain columns:
+    // - obfuscate IPs
+    // - optionally filter cookies
+    const selectExpr = event.columnNames.reduce(
+      (previous, current) => buildQueryColumns(event.columnTransformations, previous, current),
+      ''
+    );
+
+    const command = new StartQueryExecutionCommand({
+      QueryString: `
+        INSERT INTO ${database}.${targetTable} (${event.columnNames.join(',')})
+        SELECT ${selectExpr}
+        FROM ${database}.${sourceTable}
+        WHERE year = '${year}'
+          AND month = '${month}'
+          AND day = '${day}'
+          AND hour = '${hour}';`,
+      WorkGroup: workgroup,
+      QueryExecutionContext: {Database: database},
+    });
+
+    if (await executeAndAwaitResponse(command, false)) {
+      console.log('successfully transformed partition', {year, month, day, hour});
+    }
+  } catch (error) {
+    console.error('### unexpected runtime error ###', error);
+  }
+};
+
+export const handler = internalHandler;

package/lib/functions/access-logs-analysis/partitioning/tsconfig.json

@@ -0,0 +1,33 @@
+{
+  "compilerOptions": {
+    "outDir": "./build/app",
+    "target": "ES2021",
+    "module": "CommonJS",
+    "lib": ["es2021"],
+    "declaration": true,
+    "alwaysStrict": true,
+    "sourceMap": true,
+    "inlineSources": true,
+    "inlineSourceMap": false,
+    "sourceRoot": "/",
+    "noEmitHelpers": true,
+    "importHelpers": true,
+    "noImplicitAny": false,
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noImplicitReturns": true,
+    "noFallthroughCasesInSwitch": false,
+    "emitDecoratorMetadata": true,
+    "experimentalDecorators": true,
+    "strictPropertyInitialization": false,
+    "moduleResolution": "node",
+    "baseUrl": "./",
+    "paths": {
+      "*": ["./node_modules/*"]
+    },
+    "typeRoots": ["./node_modules/@types"],
+    "skipLibCheck": true
+  },
+  "include": ["./"],
+  "exclude": ["./build", "./node_modules"]
+}

package/lib/functions/access-logs-analysis/partitioning/types.d.ts

@@ -0,0 +1,7 @@
+export interface ColumnTransformationRules {
+    readonly [columnName: string]: string | undefined;
+}
+export interface TransformPartitionEvent {
+    readonly columnNames: string[];
+    readonly columnTransformations: ColumnTransformationRules;
+}

package/lib/functions/access-logs-analysis/partitioning/types.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidHlwZXMuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyJ0eXBlcy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiIiwic291cmNlc0NvbnRlbnQiOlsiZXhwb3J0IGludGVyZmFjZSBDb2x1bW5UcmFuc2Zvcm1hdGlvblJ1bGVzIHtcbiAgcmVhZG9ubHkgW2NvbHVtbk5hbWU6IHN0cmluZ106IHN0cmluZyB8IHVuZGVmaW5lZDtcbn1cblxuZXhwb3J0IGludGVyZmFjZSBUcmFuc2Zvcm1QYXJ0aXRpb25FdmVudCB7XG4gIHJlYWRvbmx5IGNvbHVtbk5hbWVzOiBzdHJpbmdbXTtcbiAgcmVhZG9ubHkgY29sdW1uVHJhbnNmb3JtYXRpb25zOiBDb2x1bW5UcmFuc2Zvcm1hdGlvblJ1bGVzO1xufVxuIl19

package/lib/functions/access-logs-analysis/partitioning/types.ts

@@ -0,0 +1,8 @@
+export interface ColumnTransformationRules {
+  readonly [columnName: string]: string | undefined;
+}
+
+export interface TransformPartitionEvent {
+  readonly columnNames: string[];
+  readonly columnTransformations: ColumnTransformationRules;
+}

package/lib/functions/access-logs-analysis/partitioning/util.d.ts

@@ -0,0 +1,9 @@
+import { StartQueryExecutionCommand } from '@aws-sdk/client-athena';
+/**
+ * Executes the given command and waits up to 10s for the command to complete.
+ * @param failOnTimeout <code>true</code> in order to throw an exception,
+ * or <code>false</code> to just log the error to the console
+ * @return <code>true</code> on success or <code>false</code> if the command timed out and
+ * <code>failOnTimeout</code> was set to false
+ */
+export declare const executeAndAwaitResponse: (command: StartQueryExecutionCommand, failOnTimeout: boolean) => Promise<boolean>;