cdk-nuxt 2.0.0 → 2.1.0

package/lib/stack/access-logs-analysis/AccessLogsAnalysis.js

@@ -0,0 +1,270 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AccessLogsAnalysis = void 0;
+const aws_s3_1 = require("aws-cdk-lib/aws-s3");
+const constructs_1 = require("constructs");
+const aws_athena_1 = require("aws-cdk-lib/aws-athena");
+const aws_lambda_1 = require("aws-cdk-lib/aws-lambda");
+const aws_events_1 = require("aws-cdk-lib/aws-events");
+const aws_cdk_lib_1 = require("aws-cdk-lib");
+const aws_glue_alpha_1 = require("@aws-cdk/aws-glue-alpha");
+const aws_lambda_event_sources_1 = require("aws-cdk-lib/aws-lambda-event-sources");
+const aws_logs_1 = require("aws-cdk-lib/aws-logs");
+const aws_iam_1 = require("aws-cdk-lib/aws-iam");
+const aws_events_targets_1 = require("aws-cdk-lib/aws-events-targets");
+const path = require("path");
+/**
+ * Provides the AWS resources to analyze access logs. This construct is derived from the official AWS sample
+ * CloudFormation stack:
+ * {@link https://github.com/aws-samples/amazon-cloudfront-access-logs-queries/blob/mainline/template.yaml}
+ */
+class AccessLogsAnalysis extends constructs_1.Construct {
+    constructor(scope, id, props) {
+        super(scope, id);
+        this.resourceIdPrefix = props.resourcePrefix;
+        this.bucket = props.bucket;
+        this.setupLifecycleRules(props);
+        this.workgroup = this.createWorkgroup();
+        this.database = this.createGlueDatabase();
+        this.accessLogsByDateTable = this.createAccessLogsByDateTable();
+        this.accessLogsParquetTable = this.createAccessLogsParquetTable();
+        this.groupByDateLayer = this.createGroupByDateLayer();
+        this.groupByDateLambda = this.createGroupByDateLambda();
+        this.partitioningLayer = this.createPartitioningLayer();
+        this.createPartitionLambda = this.createCreatePartitionLambda();
+        this.transformPartitionLambda = this.createTransformPartitionLambda();
+        this.createPartitionsScheduler = this.createCreatePartitionsScheduler();
+        this.transformPartitionsScheduler = this.createTransformPartitionsScheduler(props);
+    }
+    setupLifecycleRules(props) {
+        // cleanup raw and partitioned access logs after a configurable time range
+        var _a, _b, _c;
+        this.bucket.addLifecycleRule({
+            id: `${this.resourceIdPrefix}-cleanup-unprocessed`,
+            prefix: `${AccessLogsAnalysis.ACCESS_LOGS_FOLDER_UNPROCESSED}/*`,
+            expiration: (_a = props.expireRawLogsAfter) !== null && _a !== void 0 ? _a : aws_cdk_lib_1.Duration.days(7),
+        });
+        this.bucket.addLifecycleRule({
+            id: `${this.resourceIdPrefix}-cleanup-grouped`,
+            prefix: `${AccessLogsAnalysis.ACCESS_LOGS_FOLDER_GROUPED_BY_DATE}/*`,
+            expiration: (_b = props.expireIntermediateLogsAfter) !== null && _b !== void 0 ? _b : aws_cdk_lib_1.Duration.days(7),
+        });
+        this.bucket.addLifecycleRule({
+            id: `${this.resourceIdPrefix}-cleanup-transformed`,
+            prefix: `${AccessLogsAnalysis.ACCESS_LOGS_FOLDER_TRANSFORMED}/*`,
+            expiration: (_c = props.expireTransformedLogsAfter) !== null && _c !== void 0 ? _c : aws_cdk_lib_1.Duration.days(180),
+        });
+        /* CHECKME: delete query results after 1 week
+        this.bucket.addLifecycleRule({
+          id: `${this.resourceIdPrefix}-cleanup-query-results`,
+          prefix: `${AccessLogsAnalysis.ACCESS_LOGS_FOLDER_ATHENA_RESULTS}/*`,
+          expiration: Duration.days(7),
+        });
+        */
+    }
+    createWorkgroup() {
+        const workgroupName = `${this.resourceIdPrefix}-workgroup`;
+        // due to a current bug, the tags are not automatically assigned to the workgroup
+        // note, that the keys must be converted to lower case
+        const tags = aws_cdk_lib_1.Stack.of(this)
+            .tags.renderTags()
+            .map((tag) => ({ key: tag.Key, value: tag.Value }));
+        return new aws_athena_1.CfnWorkGroup(this, workgroupName, {
+            name: workgroupName,
+            workGroupConfiguration: {
+                publishCloudWatchMetricsEnabled: false,
+                resultConfiguration: {
+                    outputLocation: `s3://${this.bucket.bucketName}/${AccessLogsAnalysis.ACCESS_LOGS_FOLDER_ATHENA_RESULTS}`,
+                },
+                enforceWorkGroupConfiguration: true
+            },
+            tags,
+        });
+    }
+    createGlueDatabase() {
+        const databaseName = `${this.resourceIdPrefix}-database`;
+        return new aws_glue_alpha_1.Database(this, databaseName, {
+            // Athena doesn't support dashes in database/table names
+            databaseName: databaseName.replace(/-/g, '_'),
+        });
+    }
+    createGroupByDateLayer() {
+        const layerVersionName = `${this.resourceIdPrefix}-group-by-date-dependencies`;
+        return new aws_lambda_1.LayerVersion(this, layerVersionName, {
+            layerVersionName,
+            compatibleArchitectures: [aws_lambda_1.Architecture.ARM_64, aws_lambda_1.Architecture.X86_64],
+            compatibleRuntimes: [aws_lambda_1.Runtime.NODEJS_14_X, aws_lambda_1.Runtime.NODEJS_16_X],
+            code: aws_lambda_1.Code.fromAsset(path.join(__dirname, '../../functions/access-logs-analysis/group-by-date/build/layer')),
+            removalPolicy: aws_cdk_lib_1.RemovalPolicy.DESTROY,
+        });
+    }
+    createPartitioningLayer() {
+        const layerVersionName = `${this.resourceIdPrefix}-partitioning-dependencies`;
+        return new aws_lambda_1.LayerVersion(this, layerVersionName, {
+            layerVersionName,
+            compatibleArchitectures: [aws_lambda_1.Architecture.ARM_64, aws_lambda_1.Architecture.X86_64],
+            compatibleRuntimes: [aws_lambda_1.Runtime.NODEJS_14_X, aws_lambda_1.Runtime.NODEJS_16_X],
+            code: aws_lambda_1.Code.fromAsset(path.join(__dirname, '../../functions/access-logs-analysis/partitioning/build/layer')),
+            removalPolicy: aws_cdk_lib_1.RemovalPolicy.DESTROY,
+        });
+    }
+    /**
+     * A Lambda function to be triggered whenever a new access log is created.
+     * It moves the raw access logs into a sub folder hierarchy by year, month, day and hour.
+     */
+    createGroupByDateLambda() {
+        const functionName = `${this.resourceIdPrefix}-group-by-date`;
+        const lambda = new aws_lambda_1.Function(this, functionName, {
+            functionName,
+            architecture: aws_lambda_1.Architecture.ARM_64,
+            runtime: aws_lambda_1.Runtime.NODEJS_16_X,
+            code: aws_lambda_1.Code.fromAsset(path.join(__dirname, '../../functions/access-logs-analysis/group-by-date/build/app')),
+            memorySize: 512,
+            timeout: aws_cdk_lib_1.Duration.seconds(20),
+            handler: 'index.handler',
+            layers: [this.groupByDateLayer],
+            events: [
+                new aws_lambda_event_sources_1.S3EventSource(this.bucket, {
+                    events: [aws_s3_1.EventType.OBJECT_CREATED],
+                    filters: [this.getUnprocessedObjectsFilter()],
+                }),
+            ],
+            logRetention: aws_logs_1.RetentionDays.TWO_WEEKS,
+            environment: {
+                AWS_NODEJS_CONNECTION_REUSE_ENABLED: '1',
+                NODE_OPTIONS: '--enable-source-maps',
+                TARGET_FOLDER: AccessLogsAnalysis.ACCESS_LOGS_FOLDER_GROUPED_BY_DATE,
+                RAW_ACCESS_LOG_FILE_PATTERN: this.getRawAccessLogFilePattern().source,
+            },
+        });
+        this.bucket.grantReadWrite(lambda);
+        this.bucket.grantDelete(lambda, `${AccessLogsAnalysis.ACCESS_LOGS_FOLDER_UNPROCESSED}/*`);
+        return lambda;
+    }
+    /**
+     * Creates a new partition for the upcoming hour in the access log database.
+     */
+    createCreatePartitionLambda() {
+        const functionName = `${this.resourceIdPrefix}-create-part`;
+        const lambda = new aws_lambda_1.Function(this, functionName, {
+            functionName,
+            architecture: aws_lambda_1.Architecture.ARM_64,
+            runtime: aws_lambda_1.Runtime.NODEJS_16_X,
+            code: aws_lambda_1.Code.fromAsset(path.join(__dirname, '../../functions/access-logs-analysis/partitioning/build/app'), {
+                exclude: ['transform-partition*', '*.d.ts'],
+            }),
+            memorySize: 128,
+            timeout: aws_cdk_lib_1.Duration.seconds(20),
+            handler: 'create-partition.handler',
+            layers: [this.partitioningLayer],
+            logRetention: aws_logs_1.RetentionDays.TWO_WEEKS,
+            environment: {
+                AWS_NODEJS_CONNECTION_REUSE_ENABLED: '1',
+                NODE_OPTIONS: '--enable-source-maps',
+                WORKGROUP: this.workgroup.name,
+                DATABASE: this.database.databaseName,
+                TABLE: this.accessLogsByDateTable.tableName,
+            },
+        });
+        // grant access to S3 bucket and to execute Athena queries which create new partitions
+        this.bucket.grantReadWrite(lambda);
+        lambda.addToRolePolicy(new aws_iam_1.PolicyStatement({
+            effect: aws_iam_1.Effect.ALLOW,
+            actions: [
+                'athena:StartQueryExecution',
+                'athena:GetQueryExecution',
+                'glue:CreateDatabase',
+                'glue:CreatePartition',
+                'glue:GetDatabase',
+                'glue:GetTable',
+                'glue:BatchCreatePartition',
+            ],
+            resources: ['*'],
+        }));
+        return lambda;
+    }
+    /**
+     * Transforms partitions from the Hive format to Parquet.
+     */
+    createTransformPartitionLambda() {
+        const functionName = `${this.resourceIdPrefix}-transform-part`;
+        const lambda = new aws_lambda_1.Function(this, functionName, {
+            functionName,
+            architecture: aws_lambda_1.Architecture.ARM_64,
+            runtime: aws_lambda_1.Runtime.NODEJS_16_X,
+            code: aws_lambda_1.Code.fromAsset(path.join(__dirname, '../../functions/access-logs-analysis/partitioning/build/app'), {
+                exclude: ['create-partition*', '*.d.ts'],
+            }),
+            memorySize: 128,
+            timeout: aws_cdk_lib_1.Duration.seconds(20),
+            handler: 'transform-partition.handler',
+            layers: [this.partitioningLayer],
+            logRetention: aws_logs_1.RetentionDays.TWO_WEEKS,
+            environment: {
+                AWS_NODEJS_CONNECTION_REUSE_ENABLED: '1',
+                NODE_OPTIONS: '--enable-source-maps',
+                WORKGROUP: this.workgroup.name,
+                DATABASE: this.database.databaseName,
+                SOURCE_TABLE: this.accessLogsByDateTable.tableName,
+                TARGET_TABLE: this.accessLogsParquetTable.tableName,
+            },
+        });
+        // grant access to S3 bucket and to execute Athena queries which create new partitions
+        this.bucket.grantReadWrite(lambda);
+        lambda.addToRolePolicy(new aws_iam_1.PolicyStatement({
+            effect: aws_iam_1.Effect.ALLOW,
+            actions: [
+                'athena:StartQueryExecution',
+                'athena:GetQueryExecution',
+                'glue:CreateDatabase',
+                'glue:CreatePartition',
+                'glue:GetDatabase',
+                'glue:GetTable',
+                'glue:BatchCreatePartition',
+                'glue:GetPartition',
+                'glue:GetPartitions',
+                'glue:CreateTable',
+                'glue:DeleteTable',
+                'glue:DeletePartition',
+            ],
+            resources: ['*'],
+        }));
+        return lambda;
+    }
+    createCreatePartitionsScheduler() {
+        const ruleName = `${this.resourceIdPrefix}-create-part-sched`;
+        return new aws_events_1.Rule(this, ruleName, {
+            ruleName,
+            schedule: aws_events_1.Schedule.cron({ minute: '55' }),
+            targets: [new aws_events_targets_1.LambdaFunction(this.createPartitionLambda, { retryAttempts: 10 })],
+        });
+    }
+    createTransformPartitionsScheduler(props) {
+        const ruleName = `${this.resourceIdPrefix}-transform-part-sched`;
+        return new aws_events_1.Rule(this, ruleName, {
+            ruleName,
+            schedule: aws_events_1.Schedule.cron({ minute: '1' }),
+            targets: [
+                new aws_events_targets_1.LambdaFunction(this.transformPartitionLambda, {
+                    event: this.getTransformPartitionInvocationProps(props),
+                    retryAttempts: 10,
+                }),
+            ],
+        });
+    }
+    getTransformPartitionInvocationProps(props) {
+        return aws_events_1.RuleTargetInput.fromObject({
+            columnNames: [
+                ...this.accessLogsParquetTable.columns,
+                ...this.accessLogsParquetTable.partitionKeys,
+            ].map(column => column.name),
+            columnTransformations: this.getColumnTransformationRules(props),
+        });
+    }
+}
+exports.AccessLogsAnalysis = AccessLogsAnalysis;
+AccessLogsAnalysis.ACCESS_LOGS_FOLDER_UNPROCESSED = 'unprocessed';
+AccessLogsAnalysis.ACCESS_LOGS_FOLDER_GROUPED_BY_DATE = 'by-date';
+AccessLogsAnalysis.ACCESS_LOGS_FOLDER_TRANSFORMED = 'transformed';
+AccessLogsAnalysis.ACCESS_LOGS_FOLDER_ATHENA_RESULTS = 'athena-query-results';
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiQWNjZXNzTG9nc0FuYWx5c2lzLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiQWNjZXNzTG9nc0FuYWx5c2lzLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7OztBQUFBLCtDQUE0RTtBQUM1RSwyQ0FBcUM7QUFDckMsdURBQW9EO0FBR3BELHVEQUEyRjtBQUMzRix1REFBdUU7QUFDdkUsNkNBQW1FO0FBQ25FLDREQUF5RDtBQUN6RCxtRkFBbUU7QUFDbkUsbURBQW1EO0FBQ25ELGlEQUE0RDtBQUM1RCx1RUFBOEQ7QUFDOUQsNkJBQTZCO0FBSTdCOzs7O0dBSUc7QUFDSCxNQUFzQixrQkFBbUIsU0FBUSxzQkFBUztJQXFCdEQsWUFBc0IsS0FBZ0IsRUFBRSxFQUFVLEVBQUUsS0FBOEI7UUFDOUUsS0FBSyxDQUFDLEtBQUssRUFBRSxFQUFFLENBQUMsQ0FBQztRQUNqQixJQUFJLENBQUMsZ0JBQWdCLEdBQUcsS0FBSyxDQUFDLGNBQWMsQ0FBQztRQUM3QyxJQUFJLENBQUMsTUFBTSxHQUFHLEtBQUssQ0FBQyxNQUFNLENBQUM7UUFDM0IsSUFBSSxDQUFDLG1CQUFtQixDQUFDLEtBQUssQ0FBQyxDQUFDO1FBQ2hDLElBQUksQ0FBQyxTQUFTLEdBQUcsSUFBSSxDQUFDLGVBQWUsRUFBRSxDQUFDO1FBQ3hDLElBQUksQ0FBQyxRQUFRLEdBQUcsSUFBSSxDQUFDLGtCQUFrQixFQUFFLENBQUM7UUFDMUMsSUFBSSxDQUFDLHFCQUFxQixHQUFHLElBQUksQ0FBQywyQkFBMkIsRUFBRSxDQUFDO1FBQ2hFLElBQUksQ0FBQyxzQkFBc0IsR0FBRyxJQUFJLENBQUMsNEJBQTRCLEVBQUUsQ0FBQztRQUNsRSxJQUFJLENBQUMsZ0JBQWdCLEdBQUcsSUFBSSxDQUFDLHNCQUFzQixFQUFFLENBQUM7UUFDdEQsSUFBSSxDQUFDLGlCQUFpQixHQUFHLElBQUksQ0FBQyx1QkFBdUIsRUFBRSxDQUFDO1FBQ3hELElBQUksQ0FBQyxpQkFBaUIsR0FBRyxJQUFJLENBQUMsdUJBQXVCLEVBQUUsQ0FBQztRQUN4RCxJQUFJLENBQUMscUJBQXFCLEdBQUcsSUFBSSxDQUFDLDJCQUEyQixFQUFFLENBQUM7UUFDaEUsSUFBSSxDQUFDLHdCQUF3QixHQUFHLElBQUksQ0FBQyw4QkFBOEIsRUFBRSxDQUFDO1FBQ3RFLElBQUksQ0FBQyx5QkFBeUIsR0FBRyxJQUFJLENBQUMsK0JBQStCLEVBQUUsQ0FBQztRQUN4RSxJQUFJLENBQUMsNEJBQTRCLEdBQUcsSUFBSSxDQUFDLGtDQUFrQyxDQUFDLEtBQUssQ0FBQyxDQUFDO0lBQ3ZGLENBQUM7SUFFTyxtQkFBbUIsQ0FBQyxLQUE4QjtRQUN0RCwwRUFBMEU7O1FBRTFFLElBQUksQ0FBQyxNQUFNLENBQUMsZ0JBQWdCLENBQUM7WUFDekIsRUFBRSxFQUFFLEdBQUcsSUFBSSxDQUFDLGdCQUFnQixzQkFBc0I7WUFDbEQsTUFBTSxFQUFFLEdBQUcsa0JBQWtCLENBQUMsOEJBQThCLElBQUk7WUFDaEUsVUFBVSxFQUFFLE1BQUEsS0FBSyxDQUFDLGtCQUFrQixtQ0FBSSxzQkFBUSxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUM7U0FDM0QsQ0FBQyxDQUFDO1FBRUgsSUFBSSxDQUFDLE1BQU0sQ0FBQyxnQkFBZ0IsQ0FBQztZQUN6QixFQUFFLEVBQUUsR0FBRyxJQUFJLENBQUMsZ0JBQWdCLGtCQUFrQjtZQUM5QyxNQUFNLEVBQUUsR0FBRyxrQkFBa0IsQ0FBQyxrQ0FBa0MsSUFBSTtZQUNwRSxVQUFVLEVBQUUsTUFBQSxLQUFLLENBQUMsMkJBQTJCLG1DQUFJLHNCQUFRLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQztTQUNwRSxDQUFDLENBQUM7UUFFSCxJQUFJLENBQUMsTUFBTSxDQUFDLGdCQUFnQixDQUFDO1lBQ3pCLEVBQUUsRUFBRSxHQUFHLElBQUksQ0FBQyxnQkFBZ0Isc0JBQXNCO1lBQ2xELE1BQU0sRUFBRSxHQUFHLGtCQUFrQixDQUFDLDhCQUE4QixJQUFJO1lBQ2hFLFVBQVUsRUFBRSxNQUFBLEtBQUssQ0FBQywwQkFBMEIsbUNBQUksc0JBQVEsQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDO1NBQ3JFLENBQUMsQ0FBQztRQUVIOzs7Ozs7VUFNRTtJQUNOLENBQUM7SUFFTyxlQUFlO1FBQ25CLE1BQU0sYUFBYSxHQUFHLEdBQUcsSUFBSSxDQUFDLGdCQUFnQixZQUFZLENBQUM7UUFFM0QsaUZBQWlGO1FBQ2pGLHNEQUFzRDtRQUN0RCxNQUFNLElBQUksR0FBYSxtQkFBSyxDQUFDLEVBQUUsQ0FBQyxJQUFJLENBQUM7YUFDaEMsSUFBSSxDQUFDLFVBQVUsRUFBRTthQUNqQixHQUFHLENBQUMsQ0FBQyxHQUFtQyxFQUFFLEVBQUUsQ0FBQyxDQUFDLEVBQUMsR0FBRyxFQUFFLEdBQUcsQ0FBQyxHQUFHLEVBQUUsS0FBSyxFQUFFLEdBQUcsQ0FBQyxLQUFLLEVBQUMsQ0FBQyxDQUFDLENBQUM7UUFFdEYsT0FBTyxJQUFJLHlCQUFZLENBQUMsSUFBSSxFQUFFLGFBQWEsRUFBRTtZQUN6QyxJQUFJLEVBQUUsYUFBYTtZQUNuQixzQkFBc0IsRUFBRTtnQkFDcEIsK0JBQStCLEVBQUUsS0FBSztnQkFDdEMsbUJBQW1CLEVBQUU7b0JBQ2pCLGNBQWMsRUFBRSxRQUFRLElBQUksQ0FBQyxNQUFNLENBQUMsVUFBVSxJQUFJLGtCQUFrQixDQUFDLGlDQUFpQyxFQUFFO2lCQUMzRztnQkFDRCw2QkFBNkIsRUFBRSxJQUFJO2FBQ3RDO1lBQ0QsSUFBSTtTQUNQLENBQUMsQ0FBQztJQUNQLENBQUM7SUFFTyxrQkFBa0I7UUFDdEIsTUFBTSxZQUFZLEdBQUcsR0FBRyxJQUFJLENBQUMsZ0JBQWdCLFdBQVcsQ0FBQztRQUN6RCxPQUFPLElBQUkseUJBQVEsQ0FBQyxJQUFJLEVBQUUsWUFBWSxFQUFFO1lBQ3BDLHdEQUF3RDtZQUN4RCxZQUFZLEVBQUUsWUFBWSxDQUFDLE9BQU8sQ0FBQyxJQUFJLEVBQUUsR0FBRyxDQUFDO1NBQ2hELENBQUMsQ0FBQztJQUNQLENBQUM7SUFNTyxzQkFBc0I7UUFDMUIsTUFBTSxnQkFBZ0IsR0FBRyxHQUFHLElBQUksQ0FBQyxnQkFBZ0IsNkJBQTZCLENBQUM7UUFDL0UsT0FBTyxJQUFJLHlCQUFZLENBQUMsSUFBSSxFQUFFLGdCQUFnQixFQUFFO1lBQzVDLGdCQUFnQjtZQUNoQix1QkFBdUIsRUFBRSxDQUFDLHlCQUFZLENBQUMsTUFBTSxFQUFFLHlCQUFZLENBQUMsTUFBTSxDQUFDO1lBQ25FLGtCQUFrQixFQUFFLENBQUMsb0JBQU8sQ0FBQyxXQUFXLEVBQUUsb0JBQU8sQ0FBQyxXQUFXLENBQUM7WUFDOUQsSUFBSSxFQUFFLGlCQUFJLENBQUMsU0FBUyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsU0FBUyxFQUFFLGdFQUFnRSxDQUFDLENBQUM7WUFDNUcsYUFBYSxFQUFFLDJCQUFhLENBQUMsT0FBTztTQUN2QyxDQUFDLENBQUM7SUFDUCxDQUFDO0lBRU8sdUJBQXVCO1FBQzNCLE1BQU0sZ0JBQWdCLEdBQUcsR0FBRyxJQUFJLENBQUMsZ0JBQWdCLDRCQUE0QixDQUFDO1FBQzlFLE9BQU8sSUFBSSx5QkFBWSxDQUFDLElBQUksRUFBRSxnQkFBZ0IsRUFBRTtZQUM1QyxnQkFBZ0I7WUFDaEIsdUJBQXVCLEVBQUUsQ0FBQyx5QkFBWSxDQUFDLE1BQU0sRUFBRSx5QkFBWSxDQUFDLE1BQU0sQ0FBQztZQUNuRSxrQkFBa0IsRUFBRSxDQUFDLG9CQUFPLENBQUMsV0FBVyxFQUFFLG9CQUFPLENBQUMsV0FBVyxDQUFDO1lBQzlELElBQUksRUFBRSxpQkFBSSxDQUFDLFNBQVMsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLFNBQVMsRUFBRSwrREFBK0QsQ0FBQyxDQUFDO1lBQzNHLGFBQWEsRUFBRSwyQkFBYSxDQUFDLE9BQU87U0FDdkMsQ0FBQyxDQUFDO0lBQ1AsQ0FBQztJQUVEOzs7T0FHRztJQUNLLHVCQUF1QjtRQUMzQixNQUFNLFlBQVksR0FBRyxHQUFHLElBQUksQ0FBQyxnQkFBZ0IsZ0JBQWdCLENBQUM7UUFFOUQsTUFBTSxNQUFNLEdBQUcsSUFBSSxxQkFBUSxDQUFDLElBQUksRUFBRSxZQUFZLEVBQUU7WUFDNUMsWUFBWTtZQUNaLFlBQVksRUFBRSx5QkFBWSxDQUFDLE1BQU07WUFDakMsT0FBTyxFQUFFLG9CQUFPLENBQUMsV0FBVztZQUM1QixJQUFJLEVBQUUsaUJBQUksQ0FBQyxTQUFTLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxTQUFTLEVBQUUsOERBQThELENBQUMsQ0FBQztZQUMxRyxVQUFVLEVBQUUsR0FBRztZQUNmLE9BQU8sRUFBRSxzQkFBUSxDQUFDLE9BQU8sQ0FBQyxFQUFFLENBQUM7WUFDN0IsT0FBTyxFQUFFLGVBQWU7WUFDeEIsTUFBTSxFQUFFLENBQUMsSUFBSSxDQUFDLGdCQUFnQixDQUFDO1lBQy9CLE1BQU0sRUFBRTtnQkFDSixJQUFJLHdDQUFhLENBQUMsSUFBSSxDQUFDLE1BQU0sRUFBRTtvQkFDM0IsTUFBTSxFQUFFLENBQUMsa0JBQVMsQ0FBQyxjQUFjLENBQUM7b0JBQ2xDLE9BQU8sRUFBRSxDQUFDLElBQUksQ0FBQywyQkFBMkIsRUFBRSxDQUFDO2lCQUNoRCxDQUFDO2FBQ0w7WUFDRCxZQUFZLEVBQUUsd0JBQWEsQ0FBQyxTQUFTO1lBQ3JDLFdBQVcsRUFBRTtnQkFDVCxtQ0FBbUMsRUFBRSxHQUFHO2dCQUN4QyxZQUFZLEVBQUUsc0JBQXNCO2dCQUNwQyxhQUFhLEVBQUUsa0JBQWtCLENBQUMsa0NBQWtDO2dCQUNwRSwyQkFBMkIsRUFBRSxJQUFJLENBQUMsMEJBQTBCLEVBQUUsQ0FBQyxNQUFNO2FBQ3hFO1NBQ0osQ0FBQyxDQUFDO1FBRUgsSUFBSSxDQUFDLE1BQU0sQ0FBQyxjQUFjLENBQUMsTUFBTSxDQUFDLENBQUM7UUFDbkMsSUFBSSxDQUFDLE1BQU0sQ0FBQyxXQUFXLENBQUMsTUFBTSxFQUFFLEdBQUcsa0JBQWtCLENBQUMsOEJBQThCLElBQUksQ0FBQyxDQUFDO1FBRTFGLE9BQU8sTUFBTSxDQUFDO0lBQ2xCLENBQUM7SUFPRDs7T0FFRztJQUNLLDJCQUEyQjtRQUMvQixNQUFNLFlBQVksR0FBRyxHQUFHLElBQUksQ0FBQyxnQkFBZ0IsY0FBYyxDQUFDO1FBRTVELE1BQU0sTUFBTSxHQUFHLElBQUkscUJBQVEsQ0FBQyxJQUFJLEVBQUUsWUFBWSxFQUFFO1lBQzVDLFlBQVk7WUFDWixZQUFZLEVBQUUseUJBQVksQ0FBQyxNQUFNO1lBQ2pDLE9BQU8sRUFBRSxvQkFBTyxDQUFDLFdBQVc7WUFDNUIsSUFBSSxFQUFFLGlCQUFJLENBQUMsU0FBUyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsU0FBUyxFQUFFLDZEQUE2RCxDQUFDLEVBQUU7Z0JBQ3RHLE9BQU8sRUFBRSxDQUFDLHNCQUFzQixFQUFFLFFBQVEsQ0FBQzthQUM5QyxDQUFDO1lBQ0YsVUFBVSxFQUFFLEdBQUc7WUFDZixPQUFPLEVBQUUsc0JBQVEsQ0FBQyxPQUFPLENBQUMsRUFBRSxDQUFDO1lBQzdCLE9BQU8sRUFBRSwwQkFBMEI7WUFDbkMsTUFBTSxFQUFFLENBQUMsSUFBSSxDQUFDLGlCQUFpQixDQUFDO1lBQ2hDLFlBQVksRUFBRSx3QkFBYSxDQUFDLFNBQVM7WUFDckMsV0FBVyxFQUFFO2dCQUNULG1DQUFtQyxFQUFFLEdBQUc7Z0JBQ3hDLFlBQVksRUFBRSxzQkFBc0I7Z0JBQ3BDLFNBQVMsRUFBRSxJQUFJLENBQUMsU0FBUyxDQUFDLElBQUk7Z0JBQzlCLFFBQVEsRUFBRSxJQUFJLENBQUMsUUFBUSxDQUFDLFlBQVk7Z0JBQ3BDLEtBQUssRUFBRSxJQUFJLENBQUMscUJBQXFCLENBQUMsU0FBUzthQUM5QztTQUNKLENBQUMsQ0FBQztRQUVILHNGQUFzRjtRQUN0RixJQUFJLENBQUMsTUFBTSxDQUFDLGNBQWMsQ0FBQyxNQUFNLENBQUMsQ0FBQztRQUNuQyxNQUFNLENBQUMsZUFBZSxDQUNsQixJQUFJLHlCQUFlLENBQUM7WUFDaEIsTUFBTSxFQUFFLGdCQUFNLENBQUMsS0FBSztZQUNwQixPQUFPLEVBQUU7Z0JBQ0wsNEJBQTRCO2dCQUM1QiwwQkFBMEI7Z0JBQzFCLHFCQUFxQjtnQkFDckIsc0JBQXNCO2dCQUN0QixrQkFBa0I7Z0JBQ2xCLGVBQWU7Z0JBQ2YsMkJBQTJCO2FBQzlCO1lBQ0QsU0FBUyxFQUFFLENBQUMsR0FBRyxDQUFDO1NBQ25CLENBQUMsQ0FDTCxDQUFDO1FBRUYsT0FBTyxNQUFNLENBQUM7SUFDbEIsQ0FBQztJQUVEOztPQUVHO0lBQ0ssOEJBQThCO1FBQ2xDLE1BQU0sWUFBWSxHQUFHLEdBQUcsSUFBSSxDQUFDLGdCQUFnQixpQkFBaUIsQ0FBQztRQUUvRCxNQUFNLE1BQU0sR0FBRyxJQUFJLHFCQUFRLENBQUMsSUFBSSxFQUFFLFlBQVksRUFBRTtZQUM1QyxZQUFZO1lBQ1osWUFBWSxFQUFFLHlCQUFZLENBQUMsTUFBTTtZQUNqQyxPQUFPLEVBQUUsb0JBQU8sQ0FBQyxXQUFXO1lBQzVCLElBQUksRUFBRSxpQkFBSSxDQUFDLFNBQVMsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLFNBQVMsRUFBRSw2REFBNkQsQ0FBQyxFQUFFO2dCQUN0RyxPQUFPLEVBQUUsQ0FBQyxtQkFBbUIsRUFBRSxRQUFRLENBQUM7YUFDM0MsQ0FBQztZQUNGLFVBQVUsRUFBRSxHQUFHO1lBQ2YsT0FBTyxFQUFFLHNCQUFRLENBQUMsT0FBTyxDQUFDLEVBQUUsQ0FBQztZQUM3QixPQUFPLEVBQUUsNkJBQTZCO1lBQ3RDLE1BQU0sRUFBRSxDQUFDLElBQUksQ0FBQyxpQkFBaUIsQ0FBQztZQUNoQyxZQUFZLEVBQUUsd0JBQWEsQ0FBQyxTQUFTO1lBQ3JDLFdBQVcsRUFBRTtnQkFDVCxtQ0FBbUMsRUFBRSxHQUFHO2dCQUN4QyxZQUFZLEVBQUUsc0JBQXNCO2dCQUNwQyxTQUFTLEVBQUUsSUFBSSxDQUFDLFNBQVMsQ0FBQyxJQUFJO2dCQUM5QixRQUFRLEVBQUUsSUFBSSxDQUFDLFFBQVEsQ0FBQyxZQUFZO2dCQUNwQyxZQUFZLEVBQUUsSUFBSSxDQUFDLHFCQUFxQixDQUFDLFNBQVM7Z0JBQ2xELFlBQVksRUFBRSxJQUFJLENBQUMsc0JBQXNCLENBQUMsU0FBUzthQUN0RDtTQUNKLENBQUMsQ0FBQztRQUVILHNGQUFzRjtRQUN0RixJQUFJLENBQUMsTUFBTSxDQUFDLGNBQWMsQ0FBQyxNQUFNLENBQUMsQ0FBQztRQUNuQyxNQUFNLENBQUMsZUFBZSxDQUNsQixJQUFJLHlCQUFlLENBQUM7WUFDaEIsTUFBTSxFQUFFLGdCQUFNLENBQUMsS0FBSztZQUNwQixPQUFPLEVBQUU7Z0JBQ0wsNEJBQTRCO2dCQUM1QiwwQkFBMEI7Z0JBQzFCLHFCQUFxQjtnQkFDckIsc0JBQXNCO2dCQUN0QixrQkFBa0I7Z0JBQ2xCLGVBQWU7Z0JBQ2YsMkJBQTJCO2dCQUMzQixtQkFBbUI7Z0JBQ25CLG9CQUFvQjtnQkFDcEIsa0JBQWtCO2dCQUNsQixrQkFBa0I7Z0JBQ2xCLHNCQUFzQjthQUN6QjtZQUNELFNBQVMsRUFBRSxDQUFDLEdBQUcsQ0FBQztTQUNuQixDQUFDLENBQ0wsQ0FBQztRQUVGLE9BQU8sTUFBTSxDQUFDO0lBQ2xCLENBQUM7SUFFTywrQkFBK0I7UUFDbkMsTUFBTSxRQUFRLEdBQUcsR0FBRyxJQUFJLENBQUMsZ0JBQWdCLG9CQUFvQixDQUFDO1FBQzlELE9BQU8sSUFBSSxpQkFBSSxDQUFDLElBQUksRUFBRSxRQUFRLEVBQUU7WUFDNUIsUUFBUTtZQUNSLFFBQVEsRUFBRSxxQkFBUSxDQUFDLElBQUksQ0FBQyxFQUFDLE1BQU0sRUFBRSxJQUFJLEVBQUMsQ0FBQztZQUN2QyxPQUFPLEVBQUUsQ0FBQyxJQUFJLG1DQUFjLENBQUMsSUFBSSxDQUFDLHFCQUFxQixFQUFFLEVBQUMsYUFBYSxFQUFFLEVBQUUsRUFBQyxDQUFDLENBQUM7U0FDakYsQ0FBQyxDQUFDO0lBQ1AsQ0FBQztJQUVPLGtDQUFrQyxDQUFDLEtBQThCO1FBQ3JFLE1BQU0sUUFBUSxHQUFHLEdBQUcsSUFBSSxDQUFDLGdCQUFnQix1QkFBdUIsQ0FBQztRQUNqRSxPQUFPLElBQUksaUJBQUksQ0FBQyxJQUFJLEVBQUUsUUFBUSxFQUFFO1lBQzVCLFFBQVE7WUFDUixRQUFRLEVBQUUscUJBQVEsQ0FBQyxJQUFJLENBQUMsRUFBQyxNQUFNLEVBQUUsR0FBRyxFQUFDLENBQUM7WUFDdEMsT0FBTyxFQUFFO2dCQUNMLElBQUksbUNBQWMsQ0FBQyxJQUFJLENBQUMsd0JBQXdCLEVBQUU7b0JBQzlDLEtBQUssRUFBRSxJQUFJLENBQUMsb0NBQW9DLENBQUMsS0FBSyxDQUFDO29CQUN2RCxhQUFhLEVBQUUsRUFBRTtpQkFDcEIsQ0FBQzthQUNMO1NBQ0osQ0FBQyxDQUFDO0lBQ1AsQ0FBQztJQUVPLG9DQUFvQyxDQUFDLEtBQThCO1FBQ3ZFLE9BQU8sNEJBQWUsQ0FBQyxVQUFVLENBQUM7WUFDOUIsV0FBVyxFQUFFO2dCQUNULEdBQUcsSUFBSSxDQUFDLHNCQUFzQixDQUFDLE9BQU87Z0JBQ3RDLEdBQUksSUFBSSxDQUFDLHNCQUFzQixDQUFDLGFBQTBCO2FBQzdELENBQUMsR0FBRyxDQUFDLE1BQU0sQ0FBQyxFQUFFLENBQUMsTUFBTSxDQUFDLElBQUksQ0FBQztZQUM1QixxQkFBcUIsRUFBRSxJQUFJLENBQUMsNEJBQTRCLENBQUMsS0FBSyxDQUFDO1NBQ2xFLENBQUMsQ0FBQztJQUNQLENBQUM7O0FBNVNMLGdEQW1UQztBQWpUNkIsaURBQThCLEdBQUcsYUFBYSxDQUFDO0FBQy9DLHFEQUFrQyxHQUFHLFNBQVMsQ0FBQztBQUMvQyxpREFBOEIsR0FBRyxhQUFhLENBQUM7QUFDL0Msb0RBQWlDLEdBQUcsc0JBQXNCLENBQUMiLCJzb3VyY2VzQ29udGVudCI6WyJpbXBvcnQge0J1Y2tldCwgRXZlbnRUeXBlLCBOb3RpZmljYXRpb25LZXlGaWx0ZXJ9IGZyb20gJ2F3cy1jZGstbGliL2F3cy1zMyc7XG5pbXBvcnQge0NvbnN0cnVjdH0gZnJvbSAnY29uc3RydWN0cyc7XG5pbXBvcnQge0NmbldvcmtHcm91cH0gZnJvbSAnYXdzLWNkay1saWIvYXdzLWF0aGVuYSc7XG5pbXBvcnQge0Nsb3VkRnJvbnRBY2Nlc3NMb2dzQnlEYXRlVGFibGV9IGZyb20gJy4vQ2xvdWRGcm9udEFjY2Vzc0xvZ3NCeURhdGVUYWJsZSc7XG5pbXBvcnQge0FjY2Vzc0xvZ3NQYXJxdWV0VGFibGV9IGZyb20gJy4vQWNjZXNzTG9nc1BhcnF1ZXRUYWJsZSc7XG5pbXBvcnQge0FyY2hpdGVjdHVyZSwgQ29kZSwgRnVuY3Rpb24sIExheWVyVmVyc2lvbiwgUnVudGltZX0gZnJvbSAnYXdzLWNkay1saWIvYXdzLWxhbWJkYSc7XG5pbXBvcnQge1J1bGUsIFJ1bGVUYXJnZXRJbnB1dCwgU2NoZWR1bGV9IGZyb20gJ2F3cy1jZGstbGliL2F3cy1ldmVudHMnO1xuaW1wb3J0IHtDZm5UYWcsIER1cmF0aW9uLCBSZW1vdmFsUG9saWN5LCBTdGFja30gZnJvbSAnYXdzLWNkay1saWInO1xuaW1wb3J0IHtDb2x1bW4sIERhdGFiYXNlfSBmcm9tICdAYXdzLWNkay9hd3MtZ2x1ZS1hbHBoYSc7XG5pbXBvcnQge1MzRXZlbnRTb3VyY2V9IGZyb20gJ2F3cy1jZGstbGliL2F3cy1sYW1iZGEtZXZlbnQtc291cmNlcyc7XG5pbXBvcnQge1JldGVudGlvbkRheXN9IGZyb20gJ2F3cy1jZGstbGliL2F3cy1sb2dzJztcbmltcG9ydCB7RWZmZWN0LCBQb2xpY3lTdGF0ZW1lbnR9IGZyb20gJ2F3cy1jZGstbGliL2F3cy1pYW0nO1xuaW1wb3J0IHtMYW1iZGFGdW5jdGlvbn0gZnJvbSAnYXdzLWNkay1saWIvYXdzLWV2ZW50cy10YXJnZXRzJztcbmltcG9ydCAqIGFzIHBhdGggZnJvbSAncGF0aCc7XG5pbXBvcnQge0FjY2Vzc0xvZ3NBbmFseXNpc1Byb3BzfSBmcm9tIFwiLi9BY2Nlc3NMb2dzQW5hbHlzaXNQcm9wc1wiO1xuaW1wb3J0IHtDb2x1bW5UcmFuc2Zvcm1hdGlvblJ1bGVzfSBmcm9tIFwiLi4vLi4vZnVuY3Rpb25zL2FjY2Vzcy1sb2dzLWFuYWx5c2lzL3BhcnRpdGlvbmluZy90eXBlc1wiO1xuXG4vKipcbiAqIFByb3ZpZGVzIHRoZSBBV1MgcmVzb3VyY2VzIHRvIGFuYWx5emUgYWNjZXNzIGxvZ3MuIFRoaXMgY29uc3RydWN0IGlzIGRlcml2ZWQgZnJvbSB0aGUgb2ZmaWNpYWwgQVdTIHNhbXBsZVxuICogQ2xvdWRGb3JtYXRpb24gc3RhY2s6XG4gKiB7QGxpbmsgaHR0cHM6Ly9naXRodWIuY29tL2F3cy1zYW1wbGVzL2FtYXpvbi1jbG91ZGZyb250LWFjY2Vzcy1sb2dzLXF1ZXJpZXMvYmxvYi9tYWlubGluZS90ZW1wbGF0ZS55YW1sfVxuICovXG5leHBvcnQgYWJzdHJhY3QgY2xhc3MgQWNjZXNzTG9nc0FuYWx5c2lzIGV4dGVuZHMgQ29uc3RydWN0IHtcblxuICAgIHByb3RlY3RlZCBzdGF0aWMgcmVhZG9ubHkgQUNDRVNTX0xPR1NfRk9MREVSX1VOUFJPQ0VTU0VEID0gJ3VucHJvY2Vzc2VkJztcbiAgICBwcm90ZWN0ZWQgc3RhdGljIHJlYWRvbmx5IEFDQ0VTU19MT0dTX0ZPTERFUl9HUk9VUEVEX0JZX0RBVEUgPSAnYnktZGF0ZSc7XG4gICAgcHJvdGVjdGVkIHN0YXRpYyByZWFkb25seSBBQ0NFU1NfTE9HU19GT0xERVJfVFJBTlNGT1JNRUQgPSAndHJhbnNmb3JtZWQnO1xuICAgIHByb3RlY3RlZCBzdGF0aWMgcmVhZG9ubHkgQUNDRVNTX0xPR1NfRk9MREVSX0FUSEVOQV9SRVNVTFRTID0gJ2F0aGVuYS1xdWVyeS1yZXN1bHRzJztcblxuICAgIHByb3RlY3RlZCByZWFkb25seSByZXNvdXJjZUlkUHJlZml4OiBzdHJpbmc7XG4gICAgcHJvdGVjdGVkIHJlYWRvbmx5IGJ1Y2tldDogQnVja2V0O1xuICAgIHByb3RlY3RlZCByZWFkb25seSB3b3JrZ3JvdXA6IENmbldvcmtHcm91cDtcbiAgICBwcm90ZWN0ZWQgcmVhZG9ubHkgZGF0YWJhc2U6IERhdGFiYXNlO1xuICAgIHByb3RlY3RlZCByZWFkb25seSBhY2Nlc3NMb2dzQnlEYXRlVGFibGU6IENsb3VkRnJvbnRBY2Nlc3NMb2dzQnlEYXRlVGFibGU7XG4gICAgcHJvdGVjdGVkIHJlYWRvbmx5IGFjY2Vzc0xvZ3NQYXJxdWV0VGFibGU6IEFjY2Vzc0xvZ3NQYXJxdWV0VGFibGU7XG4gICAgcHJvdGVjdGVkIHJlYWRvbmx5IGdyb3VwQnlEYXRlTGF5ZXI6IExheWVyVmVyc2lvbjtcbiAgICBwcm90ZWN0ZWQgcmVhZG9ubHkgZ3JvdXBCeURhdGVMYW1iZGE6IEZ1bmN0aW9uO1xuICAgIHByb3RlY3RlZCByZWFkb25seSBwYXJ0aXRpb25pbmdMYXllcjogTGF5ZXJWZXJzaW9uO1xuICAgIHByb3RlY3RlZCByZWFkb25seSBjcmVhdGVQYXJ0aXRpb25MYW1iZGE6IEZ1bmN0aW9uO1xuICAgIHByb3RlY3RlZCByZWFkb25seSB0cmFuc2Zvcm1QYXJ0aXRpb25MYW1iZGE6IEZ1bmN0aW9uO1xuICAgIHByb3RlY3RlZCByZWFkb25seSBjcmVhdGVQYXJ0aXRpb25zU2NoZWR1bGVyOiBSdWxlO1xuICAgIHByb3RlY3RlZCByZWFkb25seSB0cmFuc2Zvcm1QYXJ0aXRpb25zU2NoZWR1bGVyOiBSdWxlO1xuXG4gICAgcHJvdGVjdGVkIGNvbnN0cnVjdG9yKHNjb3BlOiBDb25zdHJ1Y3QsIGlkOiBzdHJpbmcsIHByb3BzOiBBY2Nlc3NMb2dzQW5hbHlzaXNQcm9wcykge1xuICAgICAgICBzdXBlcihzY29wZSwgaWQpO1xuICAgICAgICB0aGlzLnJlc291cmNlSWRQcmVmaXggPSBwcm9wcy5yZXNvdXJjZVByZWZpeDtcbiAgICAgICAgdGhpcy5idWNrZXQgPSBwcm9wcy5idWNrZXQ7XG4gICAgICAgIHRoaXMuc2V0dXBMaWZlY3ljbGVSdWxlcyhwcm9wcyk7XG4gICAgICAgIHRoaXMud29ya2dyb3VwID0gdGhpcy5jcmVhdGVXb3JrZ3JvdXAoKTtcbiAgICAgICAgdGhpcy5kYXRhYmFzZSA9IHRoaXMuY3JlYXRlR2x1ZURhdGFiYXNlKCk7XG4gICAgICAgIHRoaXMuYWNjZXNzTG9nc0J5RGF0ZVRhYmxlID0gdGhpcy5jcmVhdGVBY2Nlc3NMb2dzQnlEYXRlVGFibGUoKTtcbiAgICAgICAgdGhpcy5hY2Nlc3NMb2dzUGFycXVldFRhYmxlID0gdGhpcy5jcmVhdGVBY2Nlc3NMb2dzUGFycXVldFRhYmxlKCk7XG4gICAgICAgIHRoaXMuZ3JvdXBCeURhdGVMYXllciA9IHRoaXMuY3JlYXRlR3JvdXBCeURhdGVMYXllcigpO1xuICAgICAgICB0aGlzLmdyb3VwQnlEYXRlTGFtYmRhID0gdGhpcy5jcmVhdGVHcm91cEJ5RGF0ZUxhbWJkYSgpO1xuICAgICAgICB0aGlzLnBhcnRpdGlvbmluZ0xheWVyID0gdGhpcy5jcmVhdGVQYXJ0aXRpb25pbmdMYXllcigpO1xuICAgICAgICB0aGlzLmNyZWF0ZVBhcnRpdGlvbkxhbWJkYSA9IHRoaXMuY3JlYXRlQ3JlYXRlUGFydGl0aW9uTGFtYmRhKCk7XG4gICAgICAgIHRoaXMudHJhbnNmb3JtUGFydGl0aW9uTGFtYmRhID0gdGhpcy5jcmVhdGVUcmFuc2Zvcm1QYXJ0aXRpb25MYW1iZGEoKTtcbiAgICAgICAgdGhpcy5jcmVhdGVQYXJ0aXRpb25zU2NoZWR1bGVyID0gdGhpcy5jcmVhdGVDcmVhdGVQYXJ0aXRpb25zU2NoZWR1bGVyKCk7XG4gICAgICAgIHRoaXMudHJhbnNmb3JtUGFydGl0aW9uc1NjaGVkdWxlciA9IHRoaXMuY3JlYXRlVHJhbnNmb3JtUGFydGl0aW9uc1NjaGVkdWxlcihwcm9wcyk7XG4gICAgfVxuXG4gICAgcHJpdmF0ZSBzZXR1cExpZmVjeWNsZVJ1bGVzKHByb3BzOiBBY2Nlc3NMb2dzQW5hbHlzaXNQcm9wcyk6IHZvaWQge1xuICAgICAgICAvLyBjbGVhbnVwIHJhdyBhbmQgcGFydGl0aW9uZWQgYWNjZXNzIGxvZ3MgYWZ0ZXIgYSBjb25maWd1cmFibGUgdGltZSByYW5nZVxuXG4gICAgICAgIHRoaXMuYnVja2V0LmFkZExpZmVjeWNsZVJ1bGUoe1xuICAgICAgICAgICAgaWQ6IGAke3RoaXMucmVzb3VyY2VJZFByZWZpeH0tY2xlYW51cC11bnByb2Nlc3NlZGAsXG4gICAgICAgICAgICBwcmVmaXg6IGAke0FjY2Vzc0xvZ3NBbmFseXNpcy5BQ0NFU1NfTE9HU19GT0xERVJfVU5QUk9DRVNTRUR9LypgLFxuICAgICAgICAgICAgZXhwaXJhdGlvbjogcHJvcHMuZXhwaXJlUmF3TG9nc0FmdGVyID8/IER1cmF0aW9uLmRheXMoNyksXG4gICAgICAgIH0pO1xuXG4gICAgICAgIHRoaXMuYnVja2V0LmFkZExpZmVjeWNsZVJ1bGUoe1xuICAgICAgICAgICAgaWQ6IGAke3RoaXMucmVzb3VyY2VJZFByZWZpeH0tY2xlYW51cC1ncm91cGVkYCxcbiAgICAgICAgICAgIHByZWZpeDogYCR7QWNjZXNzTG9nc0FuYWx5c2lzLkFDQ0VTU19MT0dTX0ZPTERFUl9HUk9VUEVEX0JZX0RBVEV9LypgLFxuICAgICAgICAgICAgZXhwaXJhdGlvbjogcHJvcHMuZXhwaXJlSW50ZXJtZWRpYXRlTG9nc0FmdGVyID8/IER1cmF0aW9uLmRheXMoNyksXG4gICAgICAgIH0pO1xuXG4gICAgICAgIHRoaXMuYnVja2V0LmFkZExpZmVjeWNsZVJ1bGUoe1xuICAgICAgICAgICAgaWQ6IGAke3RoaXMucmVzb3VyY2VJZFByZWZpeH0tY2xlYW51cC10cmFuc2Zvcm1lZGAsXG4gICAgICAgICAgICBwcmVmaXg6IGAke0FjY2Vzc0xvZ3NBbmFseXNpcy5BQ0NFU1NfTE9HU19GT0xERVJfVFJBTlNGT1JNRUR9LypgLFxuICAgICAgICAgICAgZXhwaXJhdGlvbjogcHJvcHMuZXhwaXJlVHJhbnNmb3JtZWRMb2dzQWZ0ZXIgPz8gRHVyYXRpb24uZGF5cygxODApLFxuICAgICAgICB9KTtcblxuICAgICAgICAvKiBDSEVDS01FOiBkZWxldGUgcXVlcnkgcmVzdWx0cyBhZnRlciAxIHdlZWtcbiAgICAgICAgdGhpcy5idWNrZXQuYWRkTGlmZWN5Y2xlUnVsZSh7XG4gICAgICAgICAgaWQ6IGAke3RoaXMucmVzb3VyY2VJZFByZWZpeH0tY2xlYW51cC1xdWVyeS1yZXN1bHRzYCxcbiAgICAgICAgICBwcmVmaXg6IGAke0FjY2Vzc0xvZ3NBbmFseXNpcy5BQ0NFU1NfTE9HU19GT0xERVJfQVRIRU5BX1JFU1VMVFN9LypgLFxuICAgICAgICAgIGV4cGlyYXRpb246IER1cmF0aW9uLmRheXMoNyksXG4gICAgICAgIH0pO1xuICAgICAgICAqL1xuICAgIH1cblxuICAgIHByaXZhdGUgY3JlYXRlV29ya2dyb3VwKCk6IENmbldvcmtHcm91cCB7XG4gICAgICAgIGNvbnN0IHdvcmtncm91cE5hbWUgPSBgJHt0aGlzLnJlc291cmNlSWRQcmVmaXh9LXdvcmtncm91cGA7XG5cbiAgICAgICAgLy8gZHVlIHRvIGEgY3VycmVudCBidWcsIHRoZSB0YWdzIGFyZSBub3QgYXV0b21hdGljYWxseSBhc3NpZ25lZCB0byB0aGUgd29ya2dyb3VwXG4gICAgICAgIC8vIG5vdGUsIHRoYXQgdGhlIGtleXMgbXVzdCBiZSBjb252ZXJ0ZWQgdG8gbG93ZXIgY2FzZVxuICAgICAgICBjb25zdCB0YWdzOiBDZm5UYWdbXSA9IFN0YWNrLm9mKHRoaXMpXG4gICAgICAgICAgICAudGFncy5yZW5kZXJUYWdzKClcbiAgICAgICAgICAgIC5tYXAoKHRhZzogeyBLZXk6IHN0cmluZzsgVmFsdWU6IHN0cmluZyB9KSA9PiAoe2tleTogdGFnLktleSwgdmFsdWU6IHRhZy5WYWx1ZX0pKTtcblxuICAgICAgICByZXR1cm4gbmV3IENmbldvcmtHcm91cCh0aGlzLCB3b3JrZ3JvdXBOYW1lLCB7XG4gICAgICAgICAgICBuYW1lOiB3b3JrZ3JvdXBOYW1lLFxuICAgICAgICAgICAgd29ya0dyb3VwQ29uZmlndXJhdGlvbjoge1xuICAgICAgICAgICAgICAgIHB1Ymxpc2hDbG91ZFdhdGNoTWV0cmljc0VuYWJsZWQ6IGZhbHNlLFxuICAgICAgICAgICAgICAgIHJlc3VsdENvbmZpZ3VyYXRpb246IHtcbiAgICAgICAgICAgICAgICAgICAgb3V0cHV0TG9jYXRpb246IGBzMzovLyR7dGhpcy5idWNrZXQuYnVja2V0TmFtZX0vJHtBY2Nlc3NMb2dzQW5hbHlzaXMuQUNDRVNTX0xPR1NfRk9MREVSX0FUSEVOQV9SRVNVTFRTfWAsXG4gICAgICAgICAgICAgICAgfSxcbiAgICAgICAgICAgICAgICBlbmZvcmNlV29ya0dyb3VwQ29uZmlndXJhdGlvbjogdHJ1ZVxuICAgICAgICAgICAgfSxcbiAgICAgICAgICAgIHRhZ3MsXG4gICAgICAgIH0pO1xuICAgIH1cblxuICAgIHByaXZhdGUgY3JlYXRlR2x1ZURhdGFiYXNlKCk6IERhdGFiYXNlIHtcbiAgICAgICAgY29uc3QgZGF0YWJhc2VOYW1lID0gYCR7dGhpcy5yZXNvdXJjZUlkUHJlZml4fS1kYXRhYmFzZWA7XG4gICAgICAgIHJldHVybiBuZXcgRGF0YWJhc2UodGhpcywgZGF0YWJhc2VOYW1lLCB7XG4gICAgICAgICAgICAvLyBBdGhlbmEgZG9lc24ndCBzdXBwb3J0IGRhc2hlcyBpbiBkYXRhYmFzZS90YWJsZSBuYW1lc1xuICAgICAgICAgICAgZGF0YWJhc2VOYW1lOiBkYXRhYmFzZU5hbWUucmVwbGFjZSgvLS9nLCAnXycpLFxuICAgICAgICB9KTtcbiAgICB9XG5cbiAgICBwcm90ZWN0ZWQgYWJzdHJhY3QgY3JlYXRlQWNjZXNzTG9nc0J5RGF0ZVRhYmxlKCk6IENsb3VkRnJvbnRBY2Nlc3NMb2dzQnlEYXRlVGFibGU7XG5cbiAgICBwcm90ZWN0ZWQgYWJzdHJhY3QgY3JlYXRlQWNjZXNzTG9nc1BhcnF1ZXRUYWJsZSgpOiBDbG91ZEZyb250QWNjZXNzTG9nc0J5RGF0ZVRhYmxlO1xuXG4gICAgcHJpdmF0ZSBjcmVhdGVHcm91cEJ5RGF0ZUxheWVyKCk6IExheWVyVmVyc2lvbiB7XG4gICAgICAgIGNvbnN0IGxheWVyVmVyc2lvbk5hbWUgPSBgJHt0aGlzLnJlc291cmNlSWRQcmVmaXh9LWdyb3VwLWJ5LWRhdGUtZGVwZW5kZW5jaWVzYDtcbiAgICAgICAgcmV0dXJuIG5ldyBMYXllclZlcnNpb24odGhpcywgbGF5ZXJWZXJzaW9uTmFtZSwge1xuICAgICAgICAgICAgbGF5ZXJWZXJzaW9uTmFtZSxcbiAgICAgICAgICAgIGNvbXBhdGlibGVBcmNoaXRlY3R1cmVzOiBbQXJjaGl0ZWN0dXJlLkFSTV82NCwgQXJjaGl0ZWN0dXJlLlg4Nl82NF0sXG4gICAgICAgICAgICBjb21wYXRpYmxlUnVudGltZXM6IFtSdW50aW1lLk5PREVKU18xNF9YLCBSdW50aW1lLk5PREVKU18xNl9YXSxcbiAgICAgICAgICAgIGNvZGU6IENvZGUuZnJvbUFzc2V0KHBhdGguam9pbihfX2Rpcm5hbWUsICcuLi8uLi9mdW5jdGlvbnMvYWNjZXNzLWxvZ3MtYW5hbHlzaXMvZ3JvdXAtYnktZGF0ZS9idWlsZC9sYXllcicpKSxcbiAgICAgICAgICAgIHJlbW92YWxQb2xpY3k6IFJlbW92YWxQb2xpY3kuREVTVFJPWSxcbiAgICAgICAgfSk7XG4gICAgfVxuXG4gICAgcHJpdmF0ZSBjcmVhdGVQYXJ0aXRpb25pbmdMYXllcigpOiBMYXllclZlcnNpb24ge1xuICAgICAgICBjb25zdCBsYXllclZlcnNpb25OYW1lID0gYCR7dGhpcy5yZXNvdXJjZUlkUHJlZml4fS1wYXJ0aXRpb25pbmctZGVwZW5kZW5jaWVzYDtcbiAgICAgICAgcmV0dXJuIG5ldyBMYXllclZlcnNpb24odGhpcywgbGF5ZXJWZXJzaW9uTmFtZSwge1xuICAgICAgICAgICAgbGF5ZXJWZXJzaW9uTmFtZSxcbiAgICAgICAgICAgIGNvbXBhdGlibGVBcmNoaXRlY3R1cmVzOiBbQXJjaGl0ZWN0dXJlLkFSTV82NCwgQXJjaGl0ZWN0dXJlLlg4Nl82NF0sXG4gICAgICAgICAgICBjb21wYXRpYmxlUnVudGltZXM6IFtSdW50aW1lLk5PREVKU18xNF9YLCBSdW50aW1lLk5PREVKU18xNl9YXSxcbiAgICAgICAgICAgIGNvZGU6IENvZGUuZnJvbUFzc2V0KHBhdGguam9pbihfX2Rpcm5hbWUsICcuLi8uLi9mdW5jdGlvbnMvYWNjZXNzLWxvZ3MtYW5hbHlzaXMvcGFydGl0aW9uaW5nL2J1aWxkL2xheWVyJykpLFxuICAgICAgICAgICAgcmVtb3ZhbFBvbGljeTogUmVtb3ZhbFBvbGljeS5ERVNUUk9ZLFxuICAgICAgICB9KTtcbiAgICB9XG5cbiAgICAvKipcbiAgICAgKiBBIExhbWJkYSBmdW5jdGlvbiB0byBiZSB0cmlnZ2VyZWQgd2hlbmV2ZXIgYSBuZXcgYWNjZXNzIGxvZyBpcyBjcmVhdGVkLlxuICAgICAqIEl0IG1vdmVzIHRoZSByYXcgYWNjZXNzIGxvZ3MgaW50byBhIHN1YiBmb2xkZXIgaGllcmFyY2h5IGJ5IHllYXIsIG1vbnRoLCBkYXkgYW5kIGhvdXIuXG4gICAgICovXG4gICAgcHJpdmF0ZSBjcmVhdGVHcm91cEJ5RGF0ZUxhbWJkYSgpOiBGdW5jdGlvbiB7XG4gICAgICAgIGNvbnN0IGZ1bmN0aW9uTmFtZSA9IGAke3RoaXMucmVzb3VyY2VJZFByZWZpeH0tZ3JvdXAtYnktZGF0ZWA7XG5cbiAgICAgICAgY29uc3QgbGFtYmRhID0gbmV3IEZ1bmN0aW9uKHRoaXMsIGZ1bmN0aW9uTmFtZSwge1xuICAgICAgICAgICAgZnVuY3Rpb25OYW1lLFxuICAgICAgICAgICAgYXJjaGl0ZWN0dXJlOiBBcmNoaXRlY3R1cmUuQVJNXzY0LFxuICAgICAgICAgICAgcnVudGltZTogUnVudGltZS5OT0RFSlNfMTZfWCxcbiAgICAgICAgICAgIGNvZGU6IENvZGUuZnJvbUFzc2V0KHBhdGguam9pbihfX2Rpcm5hbWUsICcuLi8uLi9mdW5jdGlvbnMvYWNjZXNzLWxvZ3MtYW5hbHlzaXMvZ3JvdXAtYnktZGF0ZS9idWlsZC9hcHAnKSksXG4gICAgICAgICAgICBtZW1vcnlTaXplOiA1MTIsXG4gICAgICAgICAgICB0aW1lb3V0OiBEdXJhdGlvbi5zZWNvbmRzKDIwKSxcbiAgICAgICAgICAgIGhhbmRsZXI6ICdpbmRleC5oYW5kbGVyJyxcbiAgICAgICAgICAgIGxheWVyczogW3RoaXMuZ3JvdXBCeURhdGVMYXllcl0sXG4gICAgICAgICAgICBldmVudHM6IFtcbiAgICAgICAgICAgICAgICBuZXcgUzNFdmVudFNvdXJjZSh0aGlzLmJ1Y2tldCwge1xuICAgICAgICAgICAgICAgICAgICBldmVudHM6IFtFdmVudFR5cGUuT0JKRUNUX0NSRUFURURdLFxuICAgICAgICAgICAgICAgICAgICBmaWx0ZXJzOiBbdGhpcy5nZXRVbnByb2Nlc3NlZE9iamVjdHNGaWx0ZXIoKV0sXG4gICAgICAgICAgICAgICAgfSksXG4gICAgICAgICAgICBdLFxuICAgICAgICAgICAgbG9nUmV0ZW50aW9uOiBSZXRlbnRpb25EYXlzLlRXT19XRUVLUyxcbiAgICAgICAgICAgIGVudmlyb25tZW50OiB7XG4gICAgICAgICAgICAgICAgQVdTX05PREVKU19DT05ORUNUSU9OX1JFVVNFX0VOQUJMRUQ6ICcxJyxcbiAgICAgICAgICAgICAgICBOT0RFX09QVElPTlM6ICctLWVuYWJsZS1zb3VyY2UtbWFwcycsXG4gICAgICAgICAgICAgICAgVEFSR0VUX0ZPTERFUjogQWNjZXNzTG9nc0FuYWx5c2lzLkFDQ0VTU19MT0dTX0ZPTERFUl9HUk9VUEVEX0JZX0RBVEUsXG4gICAgICAgICAgICAgICAgUkFXX0FDQ0VTU19MT0dfRklMRV9QQVRURVJOOiB0aGlzLmdldFJhd0FjY2Vzc0xvZ0ZpbGVQYXR0ZXJuKCkuc291cmNlLFxuICAgICAgICAgICAgfSxcbiAgICAgICAgfSk7XG5cbiAgICAgICAgdGhpcy5idWNrZXQuZ3JhbnRSZWFkV3JpdGUobGFtYmRhKTtcbiAgICAgICAgdGhpcy5idWNrZXQuZ3JhbnREZWxldGUobGFtYmRhLCBgJHtBY2Nlc3NMb2dzQW5hbHlzaXMuQUNDRVNTX0xPR1NfRk9MREVSX1VOUFJPQ0VTU0VEfS8qYCk7XG5cbiAgICAgICAgcmV0dXJuIGxhbWJkYTtcbiAgICB9XG5cbiAgICBwcm90ZWN0ZWQgYWJzdHJhY3QgZ2V0VW5wcm9jZXNzZWRPYmplY3RzRmlsdGVyKCk6IE5vdGlmaWNhdGlvbktleUZpbHRlcjtcblxuICAgIC8qKiBUaGUgcGF0dGVybiB0byBpZGVudGlmeSB1bnByb2Nlc3NlZCBhY2Nlc3MgbG9ncyBkZXBlbmRzIG9uIHRoZSBwcm9kdWNpbmcgc291cmNlICovXG4gICAgcHJvdGVjdGVkIGFic3RyYWN0IGdldFJhd0FjY2Vzc0xvZ0ZpbGVQYXR0ZXJuKCk6IFJlZ0V4cDtcblxuICAgIC8qKlxuICAgICAqIENyZWF0ZXMgYSBuZXcgcGFydGl0aW9uIGZvciB0aGUgdXBjb21pbmcgaG91ciBpbiB0aGUgYWNjZXNzIGxvZyBkYXRhYmFzZS5cbiAgICAgKi9cbiAgICBwcml2YXRlIGNyZWF0ZUNyZWF0ZVBhcnRpdGlvbkxhbWJkYSgpOiBGdW5jdGlvbiB7XG4gICAgICAgIGNvbnN0IGZ1bmN0aW9uTmFtZSA9IGAke3RoaXMucmVzb3VyY2VJZFByZWZpeH0tY3JlYXRlLXBhcnRgO1xuXG4gICAgICAgIGNvbnN0IGxhbWJkYSA9IG5ldyBGdW5jdGlvbih0aGlzLCBmdW5jdGlvbk5hbWUsIHtcbiAgICAgICAgICAgIGZ1bmN0aW9uTmFtZSxcbiAgICAgICAgICAgIGFyY2hpdGVjdHVyZTogQXJjaGl0ZWN0dXJlLkFSTV82NCxcbiAgICAgICAgICAgIHJ1bnRpbWU6IFJ1bnRpbWUuTk9ERUpTXzE2X1gsXG4gICAgICAgICAgICBjb2RlOiBDb2RlLmZyb21Bc3NldChwYXRoLmpvaW4oX19kaXJuYW1lLCAnLi4vLi4vZnVuY3Rpb25zL2FjY2Vzcy1sb2dzLWFuYWx5c2lzL3BhcnRpdGlvbmluZy9idWlsZC9hcHAnKSwge1xuICAgICAgICAgICAgICAgIGV4Y2x1ZGU6IFsndHJhbnNmb3JtLXBhcnRpdGlvbionLCAnKi5kLnRzJ10sXG4gICAgICAgICAgICB9KSxcbiAgICAgICAgICAgIG1lbW9yeVNpemU6IDEyOCxcbiAgICAgICAgICAgIHRpbWVvdXQ6IER1cmF0aW9uLnNlY29uZHMoMjApLFxuICAgICAgICAgICAgaGFuZGxlcjogJ2NyZWF0ZS1wYXJ0aXRpb24uaGFuZGxlcicsXG4gICAgICAgICAgICBsYXllcnM6IFt0aGlzLnBhcnRpdGlvbmluZ0xheWVyXSxcbiAgICAgICAgICAgIGxvZ1JldGVudGlvbjogUmV0ZW50aW9uRGF5cy5UV09fV0VFS1MsXG4gICAgICAgICAgICBlbnZpcm9ubWVudDoge1xuICAgICAgICAgICAgICAgIEFXU19OT0RFSlNfQ09OTkVDVElPTl9SRVVTRV9FTkFCTEVEOiAnMScsXG4gICAgICAgICAgICAgICAgTk9ERV9PUFRJT05TOiAnLS1lbmFibGUtc291cmNlLW1hcHMnLFxuICAgICAgICAgICAgICAgIFdPUktHUk9VUDogdGhpcy53b3JrZ3JvdXAubmFtZSxcbiAgICAgICAgICAgICAgICBEQVRBQkFTRTogdGhpcy5kYXRhYmFzZS5kYXRhYmFzZU5hbWUsXG4gICAgICAgICAgICAgICAgVEFCTEU6IHRoaXMuYWNjZXNzTG9nc0J5RGF0ZVRhYmxlLnRhYmxlTmFtZSxcbiAgICAgICAgICAgIH0sXG4gICAgICAgIH0pO1xuXG4gICAgICAgIC8vIGdyYW50IGFjY2VzcyB0byBTMyBidWNrZXQgYW5kIHRvIGV4ZWN1dGUgQXRoZW5hIHF1ZXJpZXMgd2hpY2ggY3JlYXRlIG5ldyBwYXJ0aXRpb25zXG4gICAgICAgIHRoaXMuYnVja2V0LmdyYW50UmVhZFdyaXRlKGxhbWJkYSk7XG4gICAgICAgIGxhbWJkYS5hZGRUb1JvbGVQb2xpY3koXG4gICAgICAgICAgICBuZXcgUG9saWN5U3RhdGVtZW50KHtcbiAgICAgICAgICAgICAgICBlZmZlY3Q6IEVmZmVjdC5BTExPVyxcbiAgICAgICAgICAgICAgICBhY3Rpb25zOiBbXG4gICAgICAgICAgICAgICAgICAgICdhdGhlbmE6U3RhcnRRdWVyeUV4ZWN1dGlvbicsXG4gICAgICAgICAgICAgICAgICAgICdhdGhlbmE6R2V0UXVlcnlFeGVjdXRpb24nLFxuICAgICAgICAgICAgICAgICAgICAnZ2x1ZTpDcmVhdGVEYXRhYmFzZScsXG4gICAgICAgICAgICAgICAgICAgICdnbHVlOkNyZWF0ZVBhcnRpdGlvbicsXG4gICAgICAgICAgICAgICAgICAgICdnbHVlOkdldERhdGFiYXNlJyxcbiAgICAgICAgICAgICAgICAgICAgJ2dsdWU6R2V0VGFibGUnLFxuICAgICAgICAgICAgICAgICAgICAnZ2x1ZTpCYXRjaENyZWF0ZVBhcnRpdGlvbicsXG4gICAgICAgICAgICAgICAgXSxcbiAgICAgICAgICAgICAgICByZXNvdXJjZXM6IFsnKiddLFxuICAgICAgICAgICAgfSlcbiAgICAgICAgKTtcblxuICAgICAgICByZXR1cm4gbGFtYmRhO1xuICAgIH1cblxuICAgIC8qKlxuICAgICAqIFRyYW5zZm9ybXMgcGFydGl0aW9ucyBmcm9tIHRoZSBIaXZlIGZvcm1hdCB0byBQYXJxdWV0LlxuICAgICAqL1xuICAgIHByaXZhdGUgY3JlYXRlVHJhbnNmb3JtUGFydGl0aW9uTGFtYmRhKCk6IEZ1bmN0aW9uIHtcbiAgICAgICAgY29uc3QgZnVuY3Rpb25OYW1lID0gYCR7dGhpcy5yZXNvdXJjZUlkUHJlZml4fS10cmFuc2Zvcm0tcGFydGA7XG5cbiAgICAgICAgY29uc3QgbGFtYmRhID0gbmV3IEZ1bmN0aW9uKHRoaXMsIGZ1bmN0aW9uTmFtZSwge1xuICAgICAgICAgICAgZnVuY3Rpb25OYW1lLFxuICAgICAgICAgICAgYXJjaGl0ZWN0dXJlOiBBcmNoaXRlY3R1cmUuQVJNXzY0LFxuICAgICAgICAgICAgcnVudGltZTogUnVudGltZS5OT0RFSlNfMTZfWCxcbiAgICAgICAgICAgIGNvZGU6IENvZGUuZnJvbUFzc2V0KHBhdGguam9pbihfX2Rpcm5hbWUsICcuLi8uLi9mdW5jdGlvbnMvYWNjZXNzLWxvZ3MtYW5hbHlzaXMvcGFydGl0aW9uaW5nL2J1aWxkL2FwcCcpLCB7XG4gICAgICAgICAgICAgICAgZXhjbHVkZTogWydjcmVhdGUtcGFydGl0aW9uKicsICcqLmQudHMnXSxcbiAgICAgICAgICAgIH0pLFxuICAgICAgICAgICAgbWVtb3J5U2l6ZTogMTI4LFxuICAgICAgICAgICAgdGltZW91dDogRHVyYXRpb24uc2Vjb25kcygyMCksXG4gICAgICAgICAgICBoYW5kbGVyOiAndHJhbnNmb3JtLXBhcnRpdGlvbi5oYW5kbGVyJyxcbiAgICAgICAgICAgIGxheWVyczogW3RoaXMucGFydGl0aW9uaW5nTGF5ZXJdLFxuICAgICAgICAgICAgbG9nUmV0ZW50aW9uOiBSZXRlbnRpb25EYXlzLlRXT19XRUVLUyxcbiAgICAgICAgICAgIGVudmlyb25tZW50OiB7XG4gICAgICAgICAgICAgICAgQVdTX05PREVKU19DT05ORUNUSU9OX1JFVVNFX0VOQUJMRUQ6ICcxJyxcbiAgICAgICAgICAgICAgICBOT0RFX09QVElPTlM6ICctLWVuYWJsZS1zb3VyY2UtbWFwcycsXG4gICAgICAgICAgICAgICAgV09SS0dST1VQOiB0aGlzLndvcmtncm91cC5uYW1lLFxuICAgICAgICAgICAgICAgIERBVEFCQVNFOiB0aGlzLmRhdGFiYXNlLmRhdGFiYXNlTmFtZSxcbiAgICAgICAgICAgICAgICBTT1VSQ0VfVEFCTEU6IHRoaXMuYWNjZXNzTG9nc0J5RGF0ZVRhYmxlLnRhYmxlTmFtZSxcbiAgICAgICAgICAgICAgICBUQVJHRVRfVEFCTEU6IHRoaXMuYWNjZXNzTG9nc1BhcnF1ZXRUYWJsZS50YWJsZU5hbWUsXG4gICAgICAgICAgICB9LFxuICAgICAgICB9KTtcblxuICAgICAgICAvLyBncmFudCBhY2Nlc3MgdG8gUzMgYnVja2V0IGFuZCB0byBleGVjdXRlIEF0aGVuYSBxdWVyaWVzIHdoaWNoIGNyZWF0ZSBuZXcgcGFydGl0aW9uc1xuICAgICAgICB0aGlzLmJ1Y2tldC5ncmFudFJlYWRXcml0ZShsYW1iZGEpO1xuICAgICAgICBsYW1iZGEuYWRkVG9Sb2xlUG9saWN5KFxuICAgICAgICAgICAgbmV3IFBvbGljeVN0YXRlbWVudCh7XG4gICAgICAgICAgICAgICAgZWZmZWN0OiBFZmZlY3QuQUxMT1csXG4gICAgICAgICAgICAgICAgYWN0aW9uczogW1xuICAgICAgICAgICAgICAgICAgICAnYXRoZW5hOlN0YXJ0UXVlcnlFeGVjdXRpb24nLFxuICAgICAgICAgICAgICAgICAgICAnYXRoZW5hOkdldFF1ZXJ5RXhlY3V0aW9uJyxcbiAgICAgICAgICAgICAgICAgICAgJ2dsdWU6Q3JlYXRlRGF0YWJhc2UnLFxuICAgICAgICAgICAgICAgICAgICAnZ2x1ZTpDcmVhdGVQYXJ0aXRpb24nLFxuICAgICAgICAgICAgICAgICAgICAnZ2x1ZTpHZXREYXRhYmFzZScsXG4gICAgICAgICAgICAgICAgICAgICdnbHVlOkdldFRhYmxlJyxcbiAgICAgICAgICAgICAgICAgICAgJ2dsdWU6QmF0Y2hDcmVhdGVQYXJ0aXRpb24nLFxuICAgICAgICAgICAgICAgICAgICAnZ2x1ZTpHZXRQYXJ0aXRpb24nLFxuICAgICAgICAgICAgICAgICAgICAnZ2x1ZTpHZXRQYXJ0aXRpb25zJyxcbiAgICAgICAgICAgICAgICAgICAgJ2dsdWU6Q3JlYXRlVGFibGUnLFxuICAgICAgICAgICAgICAgICAgICAnZ2x1ZTpEZWxldGVUYWJsZScsXG4gICAgICAgICAgICAgICAgICAgICdnbHVlOkRlbGV0ZVBhcnRpdGlvbicsXG4gICAgICAgICAgICAgICAgXSxcbiAgICAgICAgICAgICAgICByZXNvdXJjZXM6IFsnKiddLFxuICAgICAgICAgICAgfSlcbiAgICAgICAgKTtcblxuICAgICAgICByZXR1cm4gbGFtYmRhO1xuICAgIH1cblxuICAgIHByaXZhdGUgY3JlYXRlQ3JlYXRlUGFydGl0aW9uc1NjaGVkdWxlcigpOiBSdWxlIHtcbiAgICAgICAgY29uc3QgcnVsZU5hbWUgPSBgJHt0aGlzLnJlc291cmNlSWRQcmVmaXh9LWNyZWF0ZS1wYXJ0LXNjaGVkYDtcbiAgICAgICAgcmV0dXJuIG5ldyBSdWxlKHRoaXMsIHJ1bGVOYW1lLCB7XG4gICAgICAgICAgICBydWxlTmFtZSxcbiAgICAgICAgICAgIHNjaGVkdWxlOiBTY2hlZHVsZS5jcm9uKHttaW51dGU6ICc1NSd9KSxcbiAgICAgICAgICAgIHRhcmdldHM6IFtuZXcgTGFtYmRhRnVuY3Rpb24odGhpcy5jcmVhdGVQYXJ0aXRpb25MYW1iZGEsIHtyZXRyeUF0dGVtcHRzOiAxMH0pXSxcbiAgICAgICAgfSk7XG4gICAgfVxuXG4gICAgcHJpdmF0ZSBjcmVhdGVUcmFuc2Zvcm1QYXJ0aXRpb25zU2NoZWR1bGVyKHByb3BzOiBBY2Nlc3NMb2dzQW5hbHlzaXNQcm9wcyk6IFJ1bGUge1xuICAgICAgICBjb25zdCBydWxlTmFtZSA9IGAke3RoaXMucmVzb3VyY2VJZFByZWZpeH0tdHJhbnNmb3JtLXBhcnQtc2NoZWRgO1xuICAgICAgICByZXR1cm4gbmV3IFJ1bGUodGhpcywgcnVsZU5hbWUsIHtcbiAgICAgICAgICAgIHJ1bGVOYW1lLFxuICAgICAgICAgICAgc2NoZWR1bGU6IFNjaGVkdWxlLmNyb24oe21pbnV0ZTogJzEnfSksXG4gICAgICAgICAgICB0YXJnZXRzOiBbXG4gICAgICAgICAgICAgICAgbmV3IExhbWJkYUZ1bmN0aW9uKHRoaXMudHJhbnNmb3JtUGFydGl0aW9uTGFtYmRhLCB7XG4gICAgICAgICAgICAgICAgICAgIGV2ZW50OiB0aGlzLmdldFRyYW5zZm9ybVBhcnRpdGlvbkludm9jYXRpb25Qcm9wcyhwcm9wcyksXG4gICAgICAgICAgICAgICAgICAgIHJldHJ5QXR0ZW1wdHM6IDEwLFxuICAgICAgICAgICAgICAgIH0pLFxuICAgICAgICAgICAgXSxcbiAgICAgICAgfSk7XG4gICAgfVxuXG4gICAgcHJpdmF0ZSBnZXRUcmFuc2Zvcm1QYXJ0aXRpb25JbnZvY2F0aW9uUHJvcHMocHJvcHM6IEFjY2Vzc0xvZ3NBbmFseXNpc1Byb3BzKTogUnVsZVRhcmdldElucHV0IHtcbiAgICAgICAgcmV0dXJuIFJ1bGVUYXJnZXRJbnB1dC5mcm9tT2JqZWN0KHtcbiAgICAgICAgICAgIGNvbHVtbk5hbWVzOiBbXG4gICAgICAgICAgICAgICAgLi4udGhpcy5hY2Nlc3NMb2dzUGFycXVldFRhYmxlLmNvbHVtbnMsXG4gICAgICAgICAgICAgICAgLi4uKHRoaXMuYWNjZXNzTG9nc1BhcnF1ZXRUYWJsZS5wYXJ0aXRpb25LZXlzIGFzIENvbHVtbltdKSxcbiAgICAgICAgICAgIF0ubWFwKGNvbHVtbiA9PiBjb2x1bW4ubmFtZSksXG4gICAgICAgICAgICBjb2x1bW5UcmFuc2Zvcm1hdGlvbnM6IHRoaXMuZ2V0Q29sdW1uVHJhbnNmb3JtYXRpb25SdWxlcyhwcm9wcyksXG4gICAgICAgIH0pO1xuICAgIH1cblxuICAgIC8qKlxuICAgICAqIE5lZWRzIHRvIHJldHVybiB0aGUgbG9nIHNvdXJjZSBzcGVjaWZpYyBjb2x1bW4gdHJhbnNmb3JtYXRpb24gcnVsZXNcbiAgICAgKiAoZS5nLiB0byBhbm9ueW1pemUgSVAgYWRkcmVzc2VzKS5cbiAgICAgKi9cbiAgICBwcm90ZWN0ZWQgYWJzdHJhY3QgZ2V0Q29sdW1uVHJhbnNmb3JtYXRpb25SdWxlcyhwcm9wczogQWNjZXNzTG9nc0FuYWx5c2lzUHJvcHMpOiBDb2x1bW5UcmFuc2Zvcm1hdGlvblJ1bGVzO1xufVxuIl19

package/lib/stack/access-logs-analysis/AccessLogsAnalysis.ts

@@ -0,0 +1,330 @@
+import {Bucket, EventType, NotificationKeyFilter} from 'aws-cdk-lib/aws-s3';
+import {Construct} from 'constructs';
+import {CfnWorkGroup} from 'aws-cdk-lib/aws-athena';
+import {CloudFrontAccessLogsByDateTable} from './CloudFrontAccessLogsByDateTable';
+import {AccessLogsParquetTable} from './AccessLogsParquetTable';
+import {Architecture, Code, Function, LayerVersion, Runtime} from 'aws-cdk-lib/aws-lambda';
+import {Rule, RuleTargetInput, Schedule} from 'aws-cdk-lib/aws-events';
+import {CfnTag, Duration, RemovalPolicy, Stack} from 'aws-cdk-lib';
+import {Column, Database} from '@aws-cdk/aws-glue-alpha';
+import {S3EventSource} from 'aws-cdk-lib/aws-lambda-event-sources';
+import {RetentionDays} from 'aws-cdk-lib/aws-logs';
+import {Effect, PolicyStatement} from 'aws-cdk-lib/aws-iam';
+import {LambdaFunction} from 'aws-cdk-lib/aws-events-targets';
+import * as path from 'path';
+import {AccessLogsAnalysisProps} from "./AccessLogsAnalysisProps";
+import {ColumnTransformationRules} from "../../functions/access-logs-analysis/partitioning/types";
+
+/**
+ * Provides the AWS resources to analyze access logs. This construct is derived from the official AWS sample
+ * CloudFormation stack:
+ * {@link https://github.com/aws-samples/amazon-cloudfront-access-logs-queries/blob/mainline/template.yaml}
+ */
+export abstract class AccessLogsAnalysis extends Construct {
+
+    protected static readonly ACCESS_LOGS_FOLDER_UNPROCESSED = 'unprocessed';
+    protected static readonly ACCESS_LOGS_FOLDER_GROUPED_BY_DATE = 'by-date';
+    protected static readonly ACCESS_LOGS_FOLDER_TRANSFORMED = 'transformed';
+    protected static readonly ACCESS_LOGS_FOLDER_ATHENA_RESULTS = 'athena-query-results';
+
+    protected readonly resourceIdPrefix: string;
+    protected readonly bucket: Bucket;
+    protected readonly workgroup: CfnWorkGroup;
+    protected readonly database: Database;
+    protected readonly accessLogsByDateTable: CloudFrontAccessLogsByDateTable;
+    protected readonly accessLogsParquetTable: AccessLogsParquetTable;
+    protected readonly groupByDateLayer: LayerVersion;
+    protected readonly groupByDateLambda: Function;
+    protected readonly partitioningLayer: LayerVersion;
+    protected readonly createPartitionLambda: Function;
+    protected readonly transformPartitionLambda: Function;
+    protected readonly createPartitionsScheduler: Rule;
+    protected readonly transformPartitionsScheduler: Rule;
+
+    protected constructor(scope: Construct, id: string, props: AccessLogsAnalysisProps) {
+        super(scope, id);
+        this.resourceIdPrefix = props.resourcePrefix;
+        this.bucket = props.bucket;
+        this.setupLifecycleRules(props);
+        this.workgroup = this.createWorkgroup();
+        this.database = this.createGlueDatabase();
+        this.accessLogsByDateTable = this.createAccessLogsByDateTable();
+        this.accessLogsParquetTable = this.createAccessLogsParquetTable();
+        this.groupByDateLayer = this.createGroupByDateLayer();
+        this.groupByDateLambda = this.createGroupByDateLambda();
+        this.partitioningLayer = this.createPartitioningLayer();
+        this.createPartitionLambda = this.createCreatePartitionLambda();
+        this.transformPartitionLambda = this.createTransformPartitionLambda();
+        this.createPartitionsScheduler = this.createCreatePartitionsScheduler();
+        this.transformPartitionsScheduler = this.createTransformPartitionsScheduler(props);
+    }
+
+    private setupLifecycleRules(props: AccessLogsAnalysisProps): void {
+        // cleanup raw and partitioned access logs after a configurable time range
+
+        this.bucket.addLifecycleRule({
+            id: `${this.resourceIdPrefix}-cleanup-unprocessed`,
+            prefix: `${AccessLogsAnalysis.ACCESS_LOGS_FOLDER_UNPROCESSED}/*`,
+            expiration: props.expireRawLogsAfter ?? Duration.days(7),
+        });
+
+        this.bucket.addLifecycleRule({
+            id: `${this.resourceIdPrefix}-cleanup-grouped`,
+            prefix: `${AccessLogsAnalysis.ACCESS_LOGS_FOLDER_GROUPED_BY_DATE}/*`,
+            expiration: props.expireIntermediateLogsAfter ?? Duration.days(7),
+        });
+
+        this.bucket.addLifecycleRule({
+            id: `${this.resourceIdPrefix}-cleanup-transformed`,
+            prefix: `${AccessLogsAnalysis.ACCESS_LOGS_FOLDER_TRANSFORMED}/*`,
+            expiration: props.expireTransformedLogsAfter ?? Duration.days(180),
+        });
+
+        /* CHECKME: delete query results after 1 week
+        this.bucket.addLifecycleRule({
+          id: `${this.resourceIdPrefix}-cleanup-query-results`,
+          prefix: `${AccessLogsAnalysis.ACCESS_LOGS_FOLDER_ATHENA_RESULTS}/*`,
+          expiration: Duration.days(7),
+        });
+        */
+    }
+
+    private createWorkgroup(): CfnWorkGroup {
+        const workgroupName = `${this.resourceIdPrefix}-workgroup`;
+
+        // due to a current bug, the tags are not automatically assigned to the workgroup
+        // note, that the keys must be converted to lower case
+        const tags: CfnTag[] = Stack.of(this)
+            .tags.renderTags()
+            .map((tag: { Key: string; Value: string }) => ({key: tag.Key, value: tag.Value}));
+
+        return new CfnWorkGroup(this, workgroupName, {
+            name: workgroupName,
+            workGroupConfiguration: {
+                publishCloudWatchMetricsEnabled: false,
+                resultConfiguration: {
+                    outputLocation: `s3://${this.bucket.bucketName}/${AccessLogsAnalysis.ACCESS_LOGS_FOLDER_ATHENA_RESULTS}`,
+                },
+                enforceWorkGroupConfiguration: true
+            },
+            tags,
+        });
+    }
+
+    private createGlueDatabase(): Database {
+        const databaseName = `${this.resourceIdPrefix}-database`;
+        return new Database(this, databaseName, {
+            // Athena doesn't support dashes in database/table names
+            databaseName: databaseName.replace(/-/g, '_'),
+        });
+    }
+
+    protected abstract createAccessLogsByDateTable(): CloudFrontAccessLogsByDateTable;
+
+    protected abstract createAccessLogsParquetTable(): CloudFrontAccessLogsByDateTable;
+
+    private createGroupByDateLayer(): LayerVersion {
+        const layerVersionName = `${this.resourceIdPrefix}-group-by-date-dependencies`;
+        return new LayerVersion(this, layerVersionName, {
+            layerVersionName,
+            compatibleArchitectures: [Architecture.ARM_64, Architecture.X86_64],
+            compatibleRuntimes: [Runtime.NODEJS_14_X, Runtime.NODEJS_16_X],
+            code: Code.fromAsset(path.join(__dirname, '../../functions/access-logs-analysis/group-by-date/build/layer')),
+            removalPolicy: RemovalPolicy.DESTROY,
+        });
+    }
+
+    private createPartitioningLayer(): LayerVersion {
+        const layerVersionName = `${this.resourceIdPrefix}-partitioning-dependencies`;
+        return new LayerVersion(this, layerVersionName, {
+            layerVersionName,
+            compatibleArchitectures: [Architecture.ARM_64, Architecture.X86_64],
+            compatibleRuntimes: [Runtime.NODEJS_14_X, Runtime.NODEJS_16_X],
+            code: Code.fromAsset(path.join(__dirname, '../../functions/access-logs-analysis/partitioning/build/layer')),
+            removalPolicy: RemovalPolicy.DESTROY,
+        });
+    }
+
+    /**
+     * A Lambda function to be triggered whenever a new access log is created.
+     * It moves the raw access logs into a sub folder hierarchy by year, month, day and hour.
+     */
+    private createGroupByDateLambda(): Function {
+        const functionName = `${this.resourceIdPrefix}-group-by-date`;
+
+        const lambda = new Function(this, functionName, {
+            functionName,
+            architecture: Architecture.ARM_64,
+            runtime: Runtime.NODEJS_16_X,
+            code: Code.fromAsset(path.join(__dirname, '../../functions/access-logs-analysis/group-by-date/build/app')),
+            memorySize: 512,
+            timeout: Duration.seconds(20),
+            handler: 'index.handler',
+            layers: [this.groupByDateLayer],
+            events: [
+                new S3EventSource(this.bucket, {
+                    events: [EventType.OBJECT_CREATED],
+                    filters: [this.getUnprocessedObjectsFilter()],
+                }),
+            ],
+            logRetention: RetentionDays.TWO_WEEKS,
+            environment: {
+                AWS_NODEJS_CONNECTION_REUSE_ENABLED: '1',
+                NODE_OPTIONS: '--enable-source-maps',
+                TARGET_FOLDER: AccessLogsAnalysis.ACCESS_LOGS_FOLDER_GROUPED_BY_DATE,
+                RAW_ACCESS_LOG_FILE_PATTERN: this.getRawAccessLogFilePattern().source,
+            },
+        });
+
+        this.bucket.grantReadWrite(lambda);
+        this.bucket.grantDelete(lambda, `${AccessLogsAnalysis.ACCESS_LOGS_FOLDER_UNPROCESSED}/*`);
+
+        return lambda;
+    }
+
+    protected abstract getUnprocessedObjectsFilter(): NotificationKeyFilter;
+
+    /** The pattern to identify unprocessed access logs depends on the producing source */
+    protected abstract getRawAccessLogFilePattern(): RegExp;
+
+    /**
+     * Creates a new partition for the upcoming hour in the access log database.
+     */
+    private createCreatePartitionLambda(): Function {
+        const functionName = `${this.resourceIdPrefix}-create-part`;
+
+        const lambda = new Function(this, functionName, {
+            functionName,
+            architecture: Architecture.ARM_64,
+            runtime: Runtime.NODEJS_16_X,
+            code: Code.fromAsset(path.join(__dirname, '../../functions/access-logs-analysis/partitioning/build/app'), {
+                exclude: ['transform-partition*', '*.d.ts'],
+            }),
+            memorySize: 128,
+            timeout: Duration.seconds(20),
+            handler: 'create-partition.handler',
+            layers: [this.partitioningLayer],
+            logRetention: RetentionDays.TWO_WEEKS,
+            environment: {
+                AWS_NODEJS_CONNECTION_REUSE_ENABLED: '1',
+                NODE_OPTIONS: '--enable-source-maps',
+                WORKGROUP: this.workgroup.name,
+                DATABASE: this.database.databaseName,
+                TABLE: this.accessLogsByDateTable.tableName,
+            },
+        });
+
+        // grant access to S3 bucket and to execute Athena queries which create new partitions
+        this.bucket.grantReadWrite(lambda);
+        lambda.addToRolePolicy(
+            new PolicyStatement({
+                effect: Effect.ALLOW,
+                actions: [
+                    'athena:StartQueryExecution',
+                    'athena:GetQueryExecution',
+                    'glue:CreateDatabase',
+                    'glue:CreatePartition',
+                    'glue:GetDatabase',
+                    'glue:GetTable',
+                    'glue:BatchCreatePartition',
+                ],
+                resources: ['*'],
+            })
+        );
+
+        return lambda;
+    }
+
+    /**
+     * Transforms partitions from the Hive format to Parquet.
+     */
+    private createTransformPartitionLambda(): Function {
+        const functionName = `${this.resourceIdPrefix}-transform-part`;
+
+        const lambda = new Function(this, functionName, {
+            functionName,
+            architecture: Architecture.ARM_64,
+            runtime: Runtime.NODEJS_16_X,
+            code: Code.fromAsset(path.join(__dirname, '../../functions/access-logs-analysis/partitioning/build/app'), {
+                exclude: ['create-partition*', '*.d.ts'],
+            }),
+            memorySize: 128,
+            timeout: Duration.seconds(20),
+            handler: 'transform-partition.handler',
+            layers: [this.partitioningLayer],
+            logRetention: RetentionDays.TWO_WEEKS,
+            environment: {
+                AWS_NODEJS_CONNECTION_REUSE_ENABLED: '1',
+                NODE_OPTIONS: '--enable-source-maps',
+                WORKGROUP: this.workgroup.name,
+                DATABASE: this.database.databaseName,
+                SOURCE_TABLE: this.accessLogsByDateTable.tableName,
+                TARGET_TABLE: this.accessLogsParquetTable.tableName,
+            },
+        });
+
+        // grant access to S3 bucket and to execute Athena queries which create new partitions
+        this.bucket.grantReadWrite(lambda);
+        lambda.addToRolePolicy(
+            new PolicyStatement({
+                effect: Effect.ALLOW,
+                actions: [
+                    'athena:StartQueryExecution',
+                    'athena:GetQueryExecution',
+                    'glue:CreateDatabase',
+                    'glue:CreatePartition',
+                    'glue:GetDatabase',
+                    'glue:GetTable',
+                    'glue:BatchCreatePartition',
+                    'glue:GetPartition',
+                    'glue:GetPartitions',
+                    'glue:CreateTable',
+                    'glue:DeleteTable',
+                    'glue:DeletePartition',
+                ],
+                resources: ['*'],
+            })
+        );
+
+        return lambda;
+    }
+
+    private createCreatePartitionsScheduler(): Rule {
+        const ruleName = `${this.resourceIdPrefix}-create-part-sched`;
+        return new Rule(this, ruleName, {
+            ruleName,
+            schedule: Schedule.cron({minute: '55'}),
+            targets: [new LambdaFunction(this.createPartitionLambda, {retryAttempts: 10})],
+        });
+    }
+
+    private createTransformPartitionsScheduler(props: AccessLogsAnalysisProps): Rule {
+        const ruleName = `${this.resourceIdPrefix}-transform-part-sched`;
+        return new Rule(this, ruleName, {
+            ruleName,
+            schedule: Schedule.cron({minute: '1'}),
+            targets: [
+                new LambdaFunction(this.transformPartitionLambda, {
+                    event: this.getTransformPartitionInvocationProps(props),
+                    retryAttempts: 10,
+                }),
+            ],
+        });
+    }
+
+    private getTransformPartitionInvocationProps(props: AccessLogsAnalysisProps): RuleTargetInput {
+        return RuleTargetInput.fromObject({
+            columnNames: [
+                ...this.accessLogsParquetTable.columns,
+                ...(this.accessLogsParquetTable.partitionKeys as Column[]),
+            ].map(column => column.name),
+            columnTransformations: this.getColumnTransformationRules(props),
+        });
+    }
+
+    /**
+     * Needs to return the log source specific column transformation rules
+     * (e.g. to anonymize IP addresses).
+     */
+    protected abstract getColumnTransformationRules(props: AccessLogsAnalysisProps): ColumnTransformationRules;
+}