@fjall/components-infrastructure 0.100.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/lambda-assets/cert-generator/asset/index.js +17948 -0
- package/dist/lib/lambda-assets/cert-generator/asset/package.json +4 -0
- package/dist/lib/patterns/aws/clickhouseDatabase.d.ts +49 -1
- package/dist/lib/patterns/aws/clickhouseDatabase.js +137 -20
- package/dist/lib/patterns/aws/clickhouseTls/index.d.ts +1 -0
- package/dist/lib/patterns/aws/clickhouseTls/index.js +1 -0
- package/dist/lib/patterns/aws/clickhouseTls/types.d.ts +48 -0
- package/dist/lib/patterns/aws/computeEcs.d.ts +13 -1
- package/dist/lib/patterns/aws/computeEcs.js +88 -8
- package/dist/lib/patterns/aws/interfaces/database.d.ts +32 -1
- package/dist/lib/patterns/aws/interfaces/database.js +1 -1
- package/dist/lib/resources/aws/database/clickhouseConstants.d.ts +21 -0
- package/dist/lib/resources/aws/database/clickhouseConstants.js +21 -0
- package/dist/lib/resources/aws/database/clickhouseSecurityGroup.d.ts +2 -0
- package/dist/lib/resources/aws/database/clickhouseSecurityGroup.js +2 -0
- package/dist/lib/resources/aws/database/clickhouseUserData.d.ts +21 -0
- package/dist/lib/resources/aws/database/clickhouseUserData.js +48 -3
- package/dist/lib/resources/aws/database/clickhouseXmlRenderer.d.ts +1 -1
- package/dist/lib/resources/aws/database/clickhouseXmlRenderer.js +1 -1
- package/dist/lib/resources/aws/secrets/index.d.ts +2 -0
- package/dist/lib/resources/aws/secrets/index.js +2 -0
- package/dist/lib/resources/aws/secrets/tlsCaSecret.d.ts +13 -0
- package/dist/lib/resources/aws/secrets/tlsCaSecret.js +15 -0
- package/dist/lib/resources/aws/secrets/tlsServerSecret.d.ts +15 -0
- package/dist/lib/resources/aws/secrets/tlsServerSecret.js +17 -0
- package/dist/lib/resources/aws/utilities/index.d.ts +1 -0
- package/dist/lib/resources/aws/utilities/index.js +1 -0
- package/dist/lib/resources/aws/utilities/tlsCertGenerator.d.ts +33 -0
- package/dist/lib/resources/aws/utilities/tlsCertGenerator.js +67 -0
- package/package.json +7 -5
- package/dist/lib/config/aws/__t17fixture.js +0 -3
- package/dist/lib/config/aws/__t17fixtureType.d.ts +0 -2
- package/dist/lib/config/aws/__t17fixtureType.js +0 -1
- package/dist/lib/config/aws/eventBus.d.ts +0 -7
- package/dist/lib/config/aws/eventBus.js +0 -21
- package/dist/lib/config/aws/identityCenterGroupMembership.d.ts +0 -10
- package/dist/lib/config/aws/identityCenterGroupMembership.js +0 -102
- package/dist/lib/config/aws/securityBaseline.d.ts +0 -15
- package/dist/lib/config/aws/securityBaseline.js +0 -27
- package/dist/lib/patterns/aws/_eslint_test_tmp/leak.d.ts +0 -1
- package/dist/lib/patterns/aws/_eslint_test_tmp/leak.js +0 -4
- package/dist/lib/patterns/aws/managedIdentityCenter.d.ts +0 -4
- package/dist/lib/patterns/aws/managedIdentityCenter.js +0 -19
- package/dist/lib/patterns/aws/subdomainHostedZone.d.ts +0 -9
- package/dist/lib/patterns/aws/subdomainHostedZone.js +0 -34
- package/dist/lib/resources/aws/analytics/clickhouse.d.ts +0 -15
- package/dist/lib/resources/aws/analytics/clickhouse.js +0 -310
- package/dist/lib/resources/aws/analytics/clickhouseAlarms.d.ts +0 -49
- package/dist/lib/resources/aws/analytics/clickhouseAlarms.js +0 -140
- package/dist/lib/resources/aws/analytics/clickhouseConstants.d.ts +0 -73
- package/dist/lib/resources/aws/analytics/clickhouseConstants.js +0 -89
- package/dist/lib/resources/aws/analytics/clickhouseSecurityGroup.d.ts +0 -13
- package/dist/lib/resources/aws/analytics/clickhouseSecurityGroup.js +0 -28
- package/dist/lib/resources/aws/analytics/clickhouseTypes.d.ts +0 -59
- package/dist/lib/resources/aws/analytics/clickhouseTypes.js +0 -1
- package/dist/lib/resources/aws/analytics/clickhouseUserData.d.ts +0 -6
- package/dist/lib/resources/aws/analytics/clickhouseUserData.js +0 -299
- package/dist/lib/resources/aws/analytics/index.d.ts +0 -4
- package/dist/lib/resources/aws/analytics/index.js +0 -2
- package/dist/lib/resources/aws/compute/__tmp__/regression-shape.d.ts +0 -2
- package/dist/lib/resources/aws/compute/__tmp__/regression-shape.js +0 -11
- package/dist/lib/resources/aws/messaging/defaultEventBus.d.ts +0 -7
- package/dist/lib/resources/aws/messaging/defaultEventBus.js +0 -21
- package/dist/lib/resources/aws/networking/domain.d.ts +0 -13
- package/dist/lib/resources/aws/networking/domain.js +0 -100
- package/dist/lib/synth_dump.d.ts +0 -1
- package/dist/lib/synth_dump.js +0 -42
- package/dist/lib/utils/bastionFactory.d.ts +0 -10
- package/dist/lib/utils/bastionFactory.js +0 -29
- package/dist/lib/utils/constructMap.d.ts +0 -33
- package/dist/lib/utils/constructMap.js +0 -154
- package/dist/lib/utils/dnsRecords.d.ts +0 -4
- package/dist/lib/utils/dnsRecords.js +0 -104
- /package/dist/lib/{config/aws/__t17fixture.d.ts → patterns/aws/clickhouseTls/types.js} +0 -0
|
@@ -1,102 +0,0 @@
|
|
|
1
|
-
import { Fn, NestedStack } from "aws-cdk-lib";
|
|
2
|
-
import * as customResources from "aws-cdk-lib/custom-resources";
|
|
3
|
-
import { AwsCustomResource } from "../../resources/aws/utilities/awsCustomResource.js";
|
|
4
|
-
const IDENTITY_STORE_SERVICE = "identityStore";
|
|
5
|
-
const IDENTITY_CENTER_USERS_RESOURCE_TYPE = "Custom::IdentityCenterUsers";
|
|
6
|
-
// TODO: This requires a deletion and recreation to update
|
|
7
|
-
export class IdentityCenterGroupMembership extends NestedStack {
|
|
8
|
-
constructor(scope, id, props) {
|
|
9
|
-
super(scope, id);
|
|
10
|
-
const identityStoreId = Fn.importValue("identityStoreId");
|
|
11
|
-
const groupId = Fn.importValue(`${props.groupName}GroupId`);
|
|
12
|
-
for (const member of props.groupMembers) {
|
|
13
|
-
const memberSuffix = (member.split("@")[0] ?? member)
|
|
14
|
-
.split(/[^a-zA-Z0-9]/)
|
|
15
|
-
.filter((part) => part.length > 0)
|
|
16
|
-
.map((part) => part.charAt(0).toUpperCase() + part.slice(1))
|
|
17
|
-
.join("") +
|
|
18
|
-
props.groupName.charAt(0).toUpperCase() +
|
|
19
|
-
props.groupName.slice(1);
|
|
20
|
-
const listUsersCall = {
|
|
21
|
-
service: IDENTITY_STORE_SERVICE,
|
|
22
|
-
action: "listUsers",
|
|
23
|
-
parameters: {
|
|
24
|
-
IdentityStoreId: identityStoreId,
|
|
25
|
-
Filters: [
|
|
26
|
-
{
|
|
27
|
-
AttributePath: "UserName",
|
|
28
|
-
AttributeValue: member
|
|
29
|
-
}
|
|
30
|
-
]
|
|
31
|
-
},
|
|
32
|
-
physicalResourceId: customResources.PhysicalResourceId.of(`listUsers${memberSuffix}`)
|
|
33
|
-
};
|
|
34
|
-
const listUser = new AwsCustomResource(this, `ListUsersResource${memberSuffix}`, {
|
|
35
|
-
onCreate: listUsersCall,
|
|
36
|
-
onUpdate: listUsersCall
|
|
37
|
-
});
|
|
38
|
-
const userId = listUser.getResponseField("Users.0.UserId");
|
|
39
|
-
const groupMembershipId = new AwsCustomResource(this, `CreateGroupMembershipResource${memberSuffix}`, {
|
|
40
|
-
onCreate: {
|
|
41
|
-
service: IDENTITY_STORE_SERVICE,
|
|
42
|
-
action: "createGroupMembership",
|
|
43
|
-
parameters: {
|
|
44
|
-
GroupId: groupId,
|
|
45
|
-
IdentityStoreId: identityStoreId,
|
|
46
|
-
MemberId: {
|
|
47
|
-
UserId: userId
|
|
48
|
-
}
|
|
49
|
-
},
|
|
50
|
-
physicalResourceId: customResources.PhysicalResourceId.of(`createGroupMembership${memberSuffix}`)
|
|
51
|
-
},
|
|
52
|
-
resourceType: IDENTITY_CENTER_USERS_RESOURCE_TYPE
|
|
53
|
-
});
|
|
54
|
-
const refreshMembership = new AwsCustomResource(this, `RefreshMembershipResource${memberSuffix}`, {
|
|
55
|
-
onUpdate: {
|
|
56
|
-
service: IDENTITY_STORE_SERVICE,
|
|
57
|
-
action: "deleteGroupMembership",
|
|
58
|
-
parameters: {
|
|
59
|
-
IdentityStoreId: identityStoreId,
|
|
60
|
-
MembershipId: groupMembershipId.getResponseField("MembershipId")
|
|
61
|
-
},
|
|
62
|
-
physicalResourceId: customResources.PhysicalResourceId.of(`refreshGroupMembership${memberSuffix}`)
|
|
63
|
-
},
|
|
64
|
-
resourceType: IDENTITY_CENTER_USERS_RESOURCE_TYPE
|
|
65
|
-
});
|
|
66
|
-
const recreateMembership = new AwsCustomResource(this, `RecreateGroupMembershipResource${memberSuffix}`, {
|
|
67
|
-
onUpdate: {
|
|
68
|
-
service: IDENTITY_STORE_SERVICE,
|
|
69
|
-
action: "createGroupMembership",
|
|
70
|
-
parameters: {
|
|
71
|
-
GroupId: groupId,
|
|
72
|
-
IdentityStoreId: identityStoreId,
|
|
73
|
-
MemberId: {
|
|
74
|
-
UserId: userId
|
|
75
|
-
}
|
|
76
|
-
},
|
|
77
|
-
physicalResourceId: customResources.PhysicalResourceId.of(`recreateGroupMembership${memberSuffix}`),
|
|
78
|
-
// createGroupMembership throws ConflictException when the
|
|
79
|
-
// membership already exists — keeps onUpdate idempotent.
|
|
80
|
-
ignoreErrorCodesMatching: "ConflictException"
|
|
81
|
-
},
|
|
82
|
-
resourceType: IDENTITY_CENTER_USERS_RESOURCE_TYPE
|
|
83
|
-
});
|
|
84
|
-
refreshMembership.node.addDependency(recreateMembership);
|
|
85
|
-
const deleteMembership = new AwsCustomResource(this, `DeleteGroupMembershipResource${memberSuffix}`, {
|
|
86
|
-
onDelete: {
|
|
87
|
-
service: IDENTITY_STORE_SERVICE,
|
|
88
|
-
action: "deleteGroupMembership",
|
|
89
|
-
parameters: {
|
|
90
|
-
IdentityStoreId: identityStoreId,
|
|
91
|
-
MembershipId: groupMembershipId.getResponseField("MembershipId")
|
|
92
|
-
},
|
|
93
|
-
// deleteGroupMembership throws ResourceNotFoundException when
|
|
94
|
-
// the membership is already gone — keeps onDelete idempotent.
|
|
95
|
-
ignoreErrorCodesMatching: "ResourceNotFoundException"
|
|
96
|
-
},
|
|
97
|
-
resourceType: IDENTITY_CENTER_USERS_RESOURCE_TYPE
|
|
98
|
-
});
|
|
99
|
-
deleteMembership.node.addDependency(groupMembershipId);
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
}
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
import { Construct } from "constructs";
|
|
2
|
-
export interface SecurityBaselineProps {
|
|
3
|
-
/** Controls which services are instantiated. "off" creates nothing. */
|
|
4
|
-
level: "off" | "baseline" | "compliance";
|
|
5
|
-
}
|
|
6
|
-
/**
|
|
7
|
-
* Convenience orchestrator for per-account security services.
|
|
8
|
-
* Instantiates child constructs based on the selected level.
|
|
9
|
-
*
|
|
10
|
-
* Does NOT include ConfigRulePreset -- Config Rules are applied independently
|
|
11
|
-
* at the Account stack level with environment-aware presets.
|
|
12
|
-
*/
|
|
13
|
-
export declare class SecurityBaseline extends Construct {
|
|
14
|
-
constructor(scope: Construct, id: string, props: SecurityBaselineProps);
|
|
15
|
-
}
|
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
import { Construct } from "constructs";
|
|
2
|
-
import { GuardDutyDetector } from "./guardDutyDetector.js";
|
|
3
|
-
import { SecurityHubHub } from "./securityHubHub.js";
|
|
4
|
-
import { ConfigRecorder } from "./configRecorder.js";
|
|
5
|
-
import { AccountAccessAnalyser } from "./accessAnalyser.js";
|
|
6
|
-
import { InspectorEnablement } from "./inspectorEnablement.js";
|
|
7
|
-
/**
|
|
8
|
-
* Convenience orchestrator for per-account security services.
|
|
9
|
-
* Instantiates child constructs based on the selected level.
|
|
10
|
-
*
|
|
11
|
-
* Does NOT include ConfigRulePreset -- Config Rules are applied independently
|
|
12
|
-
* at the Account stack level with environment-aware presets.
|
|
13
|
-
*/
|
|
14
|
-
export class SecurityBaseline extends Construct {
|
|
15
|
-
constructor(scope, id, props) {
|
|
16
|
-
super(scope, id);
|
|
17
|
-
if (props.level === "off")
|
|
18
|
-
return;
|
|
19
|
-
new GuardDutyDetector(this, "GuardDuty");
|
|
20
|
-
new SecurityHubHub(this, "SecurityHub");
|
|
21
|
-
new ConfigRecorder(this, "Config");
|
|
22
|
-
new AccountAccessAnalyser(this, "AccessAnalyser");
|
|
23
|
-
if (props.level === "compliance") {
|
|
24
|
-
new InspectorEnablement(this, "Inspector");
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
import { Stack } from "aws-cdk-lib";
|
|
2
|
-
import * as fs from "fs";
|
|
3
|
-
import App from "../../app.js";
|
|
4
|
-
import { IdentityCenterGroupMembership } from "../../config/aws/identityCenterGroupMembership.js";
|
|
5
|
-
export class ManagedIdentityCenter extends Stack {
|
|
6
|
-
constructor(id) {
|
|
7
|
-
super(App.getInstance(), id);
|
|
8
|
-
const configFile = fs.readFileSync("../identity-center-config.json", {
|
|
9
|
-
encoding: "utf8"
|
|
10
|
-
});
|
|
11
|
-
const identityCenterConfig = JSON.parse(configFile);
|
|
12
|
-
identityCenterConfig.forEach((config) => {
|
|
13
|
-
new IdentityCenterGroupMembership(this, `Managed${config.group}Group`, {
|
|
14
|
-
groupName: config.group,
|
|
15
|
-
groupMembers: config.members
|
|
16
|
-
});
|
|
17
|
-
});
|
|
18
|
-
}
|
|
19
|
-
}
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
import { Stack } from "aws-cdk-lib";
|
|
2
|
-
export interface subdomainHostedZoneProps {
|
|
3
|
-
delegatedZone: string;
|
|
4
|
-
parentHostedZoneName: string;
|
|
5
|
-
parentAccountName: string;
|
|
6
|
-
}
|
|
7
|
-
export declare class SubdomainHostedZone extends Stack {
|
|
8
|
-
constructor(id: string, props: subdomainHostedZoneProps);
|
|
9
|
-
}
|
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
import * as route53 from "aws-cdk-lib/aws-route53";
|
|
2
|
-
import { CfnOutput, Stack } from "aws-cdk-lib";
|
|
3
|
-
import { Role } from "../../resources/aws/iam/index.js";
|
|
4
|
-
import getAccountId from "../../utils/getAccountId.js";
|
|
5
|
-
import App from "../../app.js";
|
|
6
|
-
export class SubdomainHostedZone extends Stack {
|
|
7
|
-
constructor(id, props) {
|
|
8
|
-
super(App.getInstance(), id);
|
|
9
|
-
// DelegationRoleArn
|
|
10
|
-
const delegationRoleArn = Stack.of(this).formatArn({
|
|
11
|
-
account: getAccountId(props.parentAccountName),
|
|
12
|
-
region: "",
|
|
13
|
-
resource: "role",
|
|
14
|
-
resourceName: `${props.parentHostedZoneName.split(".", 1)}DelegateHostedZoneRole`,
|
|
15
|
-
service: "iam"
|
|
16
|
-
});
|
|
17
|
-
// Delegate Hosted Zone Role
|
|
18
|
-
const hostedZoneDelegationRole = Role.fromRoleArn(this, "hostedZoneDelegationRole", delegationRoleArn);
|
|
19
|
-
// Subdomains
|
|
20
|
-
const delegatedHostedZone = new route53.HostedZone(this, `${props.delegatedZone}HostedZone`, {
|
|
21
|
-
zoneName: props.delegatedZone
|
|
22
|
-
});
|
|
23
|
-
new route53.CrossAccountZoneDelegationRecord(this, `${props.delegatedZone}DelegationRole`, {
|
|
24
|
-
delegationRole: hostedZoneDelegationRole,
|
|
25
|
-
delegatedZone: delegatedHostedZone,
|
|
26
|
-
parentHostedZoneName: props.parentHostedZoneName
|
|
27
|
-
});
|
|
28
|
-
new CfnOutput(this, `${props.delegatedZone.split(".").join("")}HostedZoneId`, {
|
|
29
|
-
key: `${props.delegatedZone.split(".").join("")}HostedZoneId`,
|
|
30
|
-
value: delegatedHostedZone.hostedZoneId,
|
|
31
|
-
exportName: `${props.delegatedZone.split(".").join("")}HostedZoneId`
|
|
32
|
-
});
|
|
33
|
-
}
|
|
34
|
-
}
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
import { Connections, type IConnectable } from "aws-cdk-lib/aws-ec2";
|
|
2
|
-
import { Construct } from "constructs";
|
|
3
|
-
import type { ClickHouseProps, ClickHouseOutputs } from "./clickhouseTypes.js";
|
|
4
|
-
/**
|
|
5
|
-
* ClickHouse analytics infrastructure.
|
|
6
|
-
*
|
|
7
|
-
* Creates a single-node ClickHouse instance on ECS EC2 with a dedicated
|
|
8
|
-
* gp3 EBS volume for data persistence. Designed for analytical workloads
|
|
9
|
-
* (cost aggregation, deployment metrics, audit logs) rather than OLTP.
|
|
10
|
-
*/
|
|
11
|
-
export default class ClickHouse extends Construct implements IConnectable {
|
|
12
|
-
readonly connections: Connections;
|
|
13
|
-
readonly outputs: ClickHouseOutputs;
|
|
14
|
-
constructor(scope: Construct, id: string, props: ClickHouseProps);
|
|
15
|
-
}
|
|
@@ -1,310 +0,0 @@
|
|
|
1
|
-
import { Cluster, Ec2TaskDefinition, NetworkMode, ContainerImage, LogDriver, AsgCapacityProvider, EcsOptimizedImage, Ec2Service, Secret as EcsSecret } from "aws-cdk-lib/aws-ecs";
|
|
2
|
-
import { ScheduledEc2Task } from "aws-cdk-lib/aws-ecs-patterns";
|
|
3
|
-
import { Schedule } from "aws-cdk-lib/aws-applicationautoscaling";
|
|
4
|
-
import { InstanceType, SubnetType, Connections, Port, UserData } from "aws-cdk-lib/aws-ec2";
|
|
5
|
-
import { AutoScalingGroup, Monitoring, BlockDeviceVolume, EbsDeviceVolumeType } from "aws-cdk-lib/aws-autoscaling";
|
|
6
|
-
import { Duration, Stack } from "aws-cdk-lib";
|
|
7
|
-
import { Construct } from "constructs";
|
|
8
|
-
import { LogGroup, RetentionDays } from "aws-cdk-lib/aws-logs";
|
|
9
|
-
import { S3Bucket } from "../storage/s3.js";
|
|
10
|
-
import { Secret } from "../secrets/secret.js";
|
|
11
|
-
import { vpcHasNatGateways } from "../../../utils/vpcUtils.js";
|
|
12
|
-
import { inferAmiHardwareType } from "../compute/ecsConstants.js";
|
|
13
|
-
import { createClickHouseSecurityGroup } from "./clickhouseSecurityGroup.js";
|
|
14
|
-
import { generateClickHouseUserData } from "./clickhouseUserData.js";
|
|
15
|
-
import { createClickHouseAlarms } from "./clickhouseAlarms.js";
|
|
16
|
-
import { CLICKHOUSE_CLUSTER_NAME, DEFAULT_CLICKHOUSE_INSTANCE_TYPE, CLICKHOUSE_IMAGE, CLICKHOUSE_EBS_VOLUME_SIZE_GB, CLICKHOUSE_EBS_IOPS, CLICKHOUSE_EBS_THROUGHPUT_MBPS, CLICKHOUSE_TASK_MEMORY_MIB, CLICKHOUSE_TASK_CPU_UNITS, CLICKHOUSE_HTTP_PORT, CLICKHOUSE_NATIVE_PORT, CLICKHOUSE_PROMETHEUS_PORT, CLICKHOUSE_DATA_MOUNT_PATH, CLICKHOUSE_SECRETS_PREFIX, CLICKHOUSE_SECRET_NAMES, CLICKHOUSE_SECRET_OPTIONS, CLICKHOUSE_HEALTH_CHECK, CLICKHOUSE_EBS_DEVICE_NAME, CLICKHOUSE_CONFIG_SUBDIR, CLICKHOUSE_USERS_SUBDIR, OPTIMISE_FINAL_SCHEDULE, REPLACING_MERGE_TREE_TABLES, OPTIMISE_MV_TABLES, CLICKHOUSE_CLOUDMAP_NAMESPACE, CLICKHOUSE_CLOUDMAP_SERVICE_NAME, OPTIMISE_TASK_MEMORY_MIB, OPTIMISE_TASK_CPU_UNITS, BACKUP_SCHEDULE, BACKUP_TASK_MEMORY_MIB, BACKUP_TASK_CPU_UNITS, BACKUP_RETENTION_DAYS } from "./clickhouseConstants.js";
|
|
17
|
-
function createClickHouseSecret(scope, id, secretKey, description) {
|
|
18
|
-
return new Secret(scope, id, {
|
|
19
|
-
secretName: `${CLICKHOUSE_SECRETS_PREFIX}/${secretKey}`,
|
|
20
|
-
description,
|
|
21
|
-
generateSecretString: CLICKHOUSE_SECRET_OPTIONS
|
|
22
|
-
});
|
|
23
|
-
}
|
|
24
|
-
/**
|
|
25
|
-
* ClickHouse analytics infrastructure.
|
|
26
|
-
*
|
|
27
|
-
* Creates a single-node ClickHouse instance on ECS EC2 with a dedicated
|
|
28
|
-
* gp3 EBS volume for data persistence. Designed for analytical workloads
|
|
29
|
-
* (cost aggregation, deployment metrics, audit logs) rather than OLTP.
|
|
30
|
-
*/
|
|
31
|
-
export default class ClickHouse extends Construct {
|
|
32
|
-
connections;
|
|
33
|
-
outputs;
|
|
34
|
-
constructor(scope, id, props) {
|
|
35
|
-
super(scope, id);
|
|
36
|
-
const contextValue = this.node.tryGetContext("clickhouseInstanceType");
|
|
37
|
-
const instanceType = (typeof contextValue === "string" ? contextValue : undefined) ??
|
|
38
|
-
props.instanceType ??
|
|
39
|
-
DEFAULT_CLICKHOUSE_INSTANCE_TYPE;
|
|
40
|
-
// 1. Security group
|
|
41
|
-
const securityGroup = createClickHouseSecurityGroup(this, props.vpc, props.webappSecurityGroup);
|
|
42
|
-
// 2. Secrets Manager secrets (auto-generated passwords)
|
|
43
|
-
const appPasswordSecret = createClickHouseSecret(this, "ClickHouseAppPassword", CLICKHOUSE_SECRET_NAMES.APP_PASSWORD, "ClickHouse application user password");
|
|
44
|
-
const auditPasswordSecret = createClickHouseSecret(this, "ClickHouseAuditPassword", CLICKHOUSE_SECRET_NAMES.AUDIT_PASSWORD, "ClickHouse audit user password");
|
|
45
|
-
const backupPasswordSecret = createClickHouseSecret(this, "ClickHouseBackupPassword", CLICKHOUSE_SECRET_NAMES.BACKUP_PASSWORD, "ClickHouse backup user password");
|
|
46
|
-
const schemaPasswordSecret = createClickHouseSecret(this, "ClickHouseSchemaPassword", CLICKHOUSE_SECRET_NAMES.SCHEMA_PASSWORD, "ClickHouse schema migration user password");
|
|
47
|
-
// 3. ECS cluster with Cloud Map namespace for service discovery
|
|
48
|
-
const cluster = new Cluster(this, "ClickHouseCluster", {
|
|
49
|
-
clusterName: CLICKHOUSE_CLUSTER_NAME,
|
|
50
|
-
vpc: props.vpc,
|
|
51
|
-
defaultCloudMapNamespace: {
|
|
52
|
-
name: CLICKHOUSE_CLOUDMAP_NAMESPACE,
|
|
53
|
-
vpc: props.vpc
|
|
54
|
-
}
|
|
55
|
-
});
|
|
56
|
-
// 4. Auto Scaling Group with gp3 EBS volume
|
|
57
|
-
const amiHardwareType = inferAmiHardwareType(instanceType);
|
|
58
|
-
const hasNat = vpcHasNatGateways(props.vpc);
|
|
59
|
-
const subnetType = hasNat
|
|
60
|
-
? SubnetType.PRIVATE_WITH_EGRESS
|
|
61
|
-
: SubnetType.PUBLIC;
|
|
62
|
-
const userData = UserData.custom(generateClickHouseUserData({
|
|
63
|
-
cfAccountId: props.r2Config?.accountId
|
|
64
|
-
}));
|
|
65
|
-
const asg = new AutoScalingGroup(this, "ClickHouseAsg", {
|
|
66
|
-
autoScalingGroupName: `${CLICKHOUSE_CLUSTER_NAME}-asg`,
|
|
67
|
-
vpc: props.vpc,
|
|
68
|
-
vpcSubnets: {
|
|
69
|
-
subnetType
|
|
70
|
-
},
|
|
71
|
-
securityGroup,
|
|
72
|
-
minCapacity: 1,
|
|
73
|
-
maxCapacity: 1,
|
|
74
|
-
desiredCapacity: 1,
|
|
75
|
-
instanceType: new InstanceType(instanceType),
|
|
76
|
-
machineImage: EcsOptimizedImage.amazonLinux2023(amiHardwareType),
|
|
77
|
-
instanceMonitoring: Monitoring.BASIC,
|
|
78
|
-
blockDevices: [
|
|
79
|
-
{
|
|
80
|
-
deviceName: CLICKHOUSE_EBS_DEVICE_NAME,
|
|
81
|
-
volume: BlockDeviceVolume.ebs(CLICKHOUSE_EBS_VOLUME_SIZE_GB, {
|
|
82
|
-
volumeType: EbsDeviceVolumeType.GP3,
|
|
83
|
-
iops: CLICKHOUSE_EBS_IOPS,
|
|
84
|
-
throughput: CLICKHOUSE_EBS_THROUGHPUT_MBPS,
|
|
85
|
-
encrypted: true
|
|
86
|
-
})
|
|
87
|
-
}
|
|
88
|
-
],
|
|
89
|
-
userData
|
|
90
|
-
});
|
|
91
|
-
// 5. Capacity provider
|
|
92
|
-
const capacityProvider = new AsgCapacityProvider(this, "ClickHouseCapacityProvider", {
|
|
93
|
-
autoScalingGroup: asg,
|
|
94
|
-
enableManagedDraining: true,
|
|
95
|
-
enableManagedTerminationProtection: false
|
|
96
|
-
});
|
|
97
|
-
cluster.addAsgCapacityProvider(capacityProvider);
|
|
98
|
-
// 6. Task definition with bind mount for EBS volume
|
|
99
|
-
const taskDefinition = new Ec2TaskDefinition(this, "ClickHouseTaskDefinition", {
|
|
100
|
-
family: CLICKHOUSE_CLUSTER_NAME,
|
|
101
|
-
networkMode: NetworkMode.AWS_VPC
|
|
102
|
-
});
|
|
103
|
-
taskDefinition.addVolume({
|
|
104
|
-
name: "clickhouse-data",
|
|
105
|
-
host: {
|
|
106
|
-
sourcePath: CLICKHOUSE_DATA_MOUNT_PATH
|
|
107
|
-
}
|
|
108
|
-
});
|
|
109
|
-
taskDefinition.addVolume({
|
|
110
|
-
name: "clickhouse-config",
|
|
111
|
-
host: {
|
|
112
|
-
sourcePath: `${CLICKHOUSE_DATA_MOUNT_PATH}/${CLICKHOUSE_CONFIG_SUBDIR}`
|
|
113
|
-
}
|
|
114
|
-
});
|
|
115
|
-
taskDefinition.addVolume({
|
|
116
|
-
name: "clickhouse-users",
|
|
117
|
-
host: {
|
|
118
|
-
sourcePath: `${CLICKHOUSE_DATA_MOUNT_PATH}/${CLICKHOUSE_USERS_SUBDIR}`
|
|
119
|
-
}
|
|
120
|
-
});
|
|
121
|
-
// 7. Container
|
|
122
|
-
const container = taskDefinition.addContainer("clickhouse", {
|
|
123
|
-
image: ContainerImage.fromRegistry(CLICKHOUSE_IMAGE),
|
|
124
|
-
memoryLimitMiB: CLICKHOUSE_TASK_MEMORY_MIB,
|
|
125
|
-
cpu: CLICKHOUSE_TASK_CPU_UNITS,
|
|
126
|
-
logging: LogDriver.awsLogs({
|
|
127
|
-
streamPrefix: "clickhouse",
|
|
128
|
-
logRetention: RetentionDays.TWO_WEEKS
|
|
129
|
-
}),
|
|
130
|
-
healthCheck: {
|
|
131
|
-
command: [
|
|
132
|
-
"CMD-SHELL",
|
|
133
|
-
`curl -f http://localhost:${CLICKHOUSE_HTTP_PORT}/?query=SELECT%201 || exit 1`
|
|
134
|
-
],
|
|
135
|
-
interval: Duration.seconds(CLICKHOUSE_HEALTH_CHECK.INTERVAL_SECONDS),
|
|
136
|
-
timeout: Duration.seconds(CLICKHOUSE_HEALTH_CHECK.TIMEOUT_SECONDS),
|
|
137
|
-
retries: CLICKHOUSE_HEALTH_CHECK.RETRIES,
|
|
138
|
-
startPeriod: Duration.seconds(CLICKHOUSE_HEALTH_CHECK.START_PERIOD_SECONDS)
|
|
139
|
-
},
|
|
140
|
-
secrets: {
|
|
141
|
-
CLICKHOUSE_APP_PASSWORD: EcsSecret.fromSecretsManager(appPasswordSecret.secret),
|
|
142
|
-
CLICKHOUSE_AUDIT_PASSWORD: EcsSecret.fromSecretsManager(auditPasswordSecret.secret),
|
|
143
|
-
...(props.r2Config
|
|
144
|
-
? {
|
|
145
|
-
R2_ACCESS_KEY: EcsSecret.fromSecretsManager(props.r2Config.accessKeySecret),
|
|
146
|
-
R2_SECRET_KEY: EcsSecret.fromSecretsManager(props.r2Config.secretKeySecret)
|
|
147
|
-
}
|
|
148
|
-
: {})
|
|
149
|
-
},
|
|
150
|
-
portMappings: [
|
|
151
|
-
{ containerPort: CLICKHOUSE_HTTP_PORT, hostPort: CLICKHOUSE_HTTP_PORT },
|
|
152
|
-
{
|
|
153
|
-
containerPort: CLICKHOUSE_NATIVE_PORT,
|
|
154
|
-
hostPort: CLICKHOUSE_NATIVE_PORT
|
|
155
|
-
},
|
|
156
|
-
{
|
|
157
|
-
containerPort: CLICKHOUSE_PROMETHEUS_PORT,
|
|
158
|
-
hostPort: CLICKHOUSE_PROMETHEUS_PORT
|
|
159
|
-
}
|
|
160
|
-
]
|
|
161
|
-
});
|
|
162
|
-
container.addMountPoints({
|
|
163
|
-
sourceVolume: "clickhouse-data",
|
|
164
|
-
containerPath: "/var/lib/clickhouse",
|
|
165
|
-
readOnly: false
|
|
166
|
-
}, {
|
|
167
|
-
sourceVolume: "clickhouse-config",
|
|
168
|
-
containerPath: "/etc/clickhouse-server/config.d",
|
|
169
|
-
readOnly: true
|
|
170
|
-
}, {
|
|
171
|
-
sourceVolume: "clickhouse-users",
|
|
172
|
-
containerPath: "/etc/clickhouse-server/users.d",
|
|
173
|
-
readOnly: true
|
|
174
|
-
});
|
|
175
|
-
// 8. ECS service with Cloud Map registration for optimise task discovery
|
|
176
|
-
const clickHouseHost = `${CLICKHOUSE_CLOUDMAP_SERVICE_NAME}.${CLICKHOUSE_CLOUDMAP_NAMESPACE}`;
|
|
177
|
-
new Ec2Service(this, "ClickHouseService", {
|
|
178
|
-
cluster,
|
|
179
|
-
taskDefinition,
|
|
180
|
-
desiredCount: 1,
|
|
181
|
-
capacityProviderStrategies: [
|
|
182
|
-
{
|
|
183
|
-
capacityProvider: capacityProvider.capacityProviderName,
|
|
184
|
-
weight: 1
|
|
185
|
-
}
|
|
186
|
-
],
|
|
187
|
-
circuitBreaker: { rollback: true },
|
|
188
|
-
cloudMapOptions: {
|
|
189
|
-
name: CLICKHOUSE_CLOUDMAP_SERVICE_NAME
|
|
190
|
-
}
|
|
191
|
-
});
|
|
192
|
-
// 9. Scheduled OPTIMIZE TABLE FINAL task (deduplicates ReplacingMergeTree tables)
|
|
193
|
-
const optimiseQuery = [
|
|
194
|
-
...REPLACING_MERGE_TREE_TABLES.map((table) => `OPTIMIZE TABLE analytics.${table} FINAL`),
|
|
195
|
-
...OPTIMISE_MV_TABLES.map((table) => `OPTIMIZE TABLE analytics.${table}`)
|
|
196
|
-
].join("; ");
|
|
197
|
-
new ScheduledEc2Task(this, "ClickHouseOptimiseTask", {
|
|
198
|
-
cluster,
|
|
199
|
-
schedule: Schedule.expression(OPTIMISE_FINAL_SCHEDULE),
|
|
200
|
-
scheduledEc2TaskImageOptions: {
|
|
201
|
-
image: ContainerImage.fromRegistry(CLICKHOUSE_IMAGE),
|
|
202
|
-
memoryLimitMiB: OPTIMISE_TASK_MEMORY_MIB,
|
|
203
|
-
cpu: OPTIMISE_TASK_CPU_UNITS,
|
|
204
|
-
command: [
|
|
205
|
-
"clickhouse-client",
|
|
206
|
-
"--host",
|
|
207
|
-
clickHouseHost,
|
|
208
|
-
"--port",
|
|
209
|
-
String(CLICKHOUSE_NATIVE_PORT),
|
|
210
|
-
"--user",
|
|
211
|
-
"schema_admin",
|
|
212
|
-
"--query",
|
|
213
|
-
`${optimiseQuery};`
|
|
214
|
-
],
|
|
215
|
-
secrets: {
|
|
216
|
-
CLICKHOUSE_PASSWORD: EcsSecret.fromSecretsManager(schemaPasswordSecret.secret)
|
|
217
|
-
},
|
|
218
|
-
logDriver: LogDriver.awsLogs({
|
|
219
|
-
streamPrefix: "clickhouse-optimise",
|
|
220
|
-
logRetention: RetentionDays.ONE_WEEK
|
|
221
|
-
})
|
|
222
|
-
},
|
|
223
|
-
securityGroups: [securityGroup],
|
|
224
|
-
subnetSelection: {
|
|
225
|
-
subnetType
|
|
226
|
-
}
|
|
227
|
-
});
|
|
228
|
-
// 10. S3 bucket for weekly backups
|
|
229
|
-
const backupBucket = new S3Bucket(this, "ClickHouseBackupBucket", {
|
|
230
|
-
versioned: true,
|
|
231
|
-
lifecycleRules: [
|
|
232
|
-
{
|
|
233
|
-
enabled: true,
|
|
234
|
-
expiration: Duration.days(BACKUP_RETENTION_DAYS),
|
|
235
|
-
noncurrentVersionExpiration: Duration.days(BACKUP_RETENTION_DAYS)
|
|
236
|
-
}
|
|
237
|
-
]
|
|
238
|
-
});
|
|
239
|
-
// 11. Scheduled weekly backup to S3
|
|
240
|
-
const backupDestUrl = `https://${backupBucket.bucketName}.s3.${Stack.of(this).region}.amazonaws.com/`;
|
|
241
|
-
const backupTaskLogGroup = new LogGroup(this, "ClickHouseBackupTaskLogGroup", {
|
|
242
|
-
retention: RetentionDays.TWO_WEEKS
|
|
243
|
-
});
|
|
244
|
-
new ScheduledEc2Task(this, "ClickHouseBackupTask", {
|
|
245
|
-
cluster,
|
|
246
|
-
schedule: Schedule.expression(BACKUP_SCHEDULE),
|
|
247
|
-
scheduledEc2TaskImageOptions: {
|
|
248
|
-
image: ContainerImage.fromRegistry(CLICKHOUSE_IMAGE),
|
|
249
|
-
memoryLimitMiB: BACKUP_TASK_MEMORY_MIB,
|
|
250
|
-
cpu: BACKUP_TASK_CPU_UNITS,
|
|
251
|
-
command: [
|
|
252
|
-
"sh",
|
|
253
|
-
"-c",
|
|
254
|
-
`STAMP=$(date +%Y%m%d-%H%M%S) && clickhouse-client --host ${clickHouseHost} --port ${CLICKHOUSE_NATIVE_PORT} --user backup_reader --password "$CLICKHOUSE_BACKUP_PASSWORD" --query "BACKUP DATABASE analytics TO S3('${backupDestUrl}weekly-$STAMP/')"`
|
|
255
|
-
],
|
|
256
|
-
secrets: {
|
|
257
|
-
CLICKHOUSE_BACKUP_PASSWORD: EcsSecret.fromSecretsManager(backupPasswordSecret.secret)
|
|
258
|
-
},
|
|
259
|
-
logDriver: LogDriver.awsLogs({
|
|
260
|
-
streamPrefix: "clickhouse-backup",
|
|
261
|
-
logGroup: backupTaskLogGroup
|
|
262
|
-
})
|
|
263
|
-
},
|
|
264
|
-
securityGroups: [securityGroup],
|
|
265
|
-
subnetSelection: {
|
|
266
|
-
subnetType
|
|
267
|
-
}
|
|
268
|
-
});
|
|
269
|
-
// BACKUP DATABASE TO S3 runs inside the ClickHouse server process on the
|
|
270
|
-
// ASG instance, not the ephemeral backup task; the grant must therefore
|
|
271
|
-
// attach to the ASG instance role, not the task role.
|
|
272
|
-
backupBucket.grantReadWrite(asg.role);
|
|
273
|
-
// 12. Grant secret read to execution role
|
|
274
|
-
const executionRole = taskDefinition.executionRole;
|
|
275
|
-
if (!executionRole) {
|
|
276
|
-
throw new Error("ClickHouse task definition has no execution role — cannot grant secret access");
|
|
277
|
-
}
|
|
278
|
-
appPasswordSecret.secret.grantRead(executionRole);
|
|
279
|
-
auditPasswordSecret.secret.grantRead(executionRole);
|
|
280
|
-
backupPasswordSecret.secret.grantRead(executionRole);
|
|
281
|
-
schemaPasswordSecret.secret.grantRead(executionRole);
|
|
282
|
-
if (props.alarmTopic) {
|
|
283
|
-
if (!props.webappLogGroup) {
|
|
284
|
-
throw new Error("ClickHouse: alarmTopic requires webappLogGroup so the stuck-merge metric filter can be wired.");
|
|
285
|
-
}
|
|
286
|
-
createClickHouseAlarms({
|
|
287
|
-
scope: this,
|
|
288
|
-
asg,
|
|
289
|
-
alarmTopic: props.alarmTopic,
|
|
290
|
-
webappLogGroup: props.webappLogGroup,
|
|
291
|
-
backupTaskLogGroup
|
|
292
|
-
});
|
|
293
|
-
}
|
|
294
|
-
// 13. Connections and outputs
|
|
295
|
-
this.connections = new Connections({
|
|
296
|
-
securityGroups: [securityGroup],
|
|
297
|
-
defaultPort: Port.tcp(CLICKHOUSE_HTTP_PORT)
|
|
298
|
-
});
|
|
299
|
-
this.outputs = {
|
|
300
|
-
securityGroup,
|
|
301
|
-
backupBucket,
|
|
302
|
-
secrets: {
|
|
303
|
-
appPassword: appPasswordSecret.secret,
|
|
304
|
-
auditPassword: auditPasswordSecret.secret,
|
|
305
|
-
backupPassword: backupPasswordSecret.secret,
|
|
306
|
-
schemaPassword: schemaPasswordSecret.secret
|
|
307
|
-
}
|
|
308
|
-
};
|
|
309
|
-
}
|
|
310
|
-
}
|
|
@@ -1,49 +0,0 @@
|
|
|
1
|
-
import { Alarm } from "aws-cdk-lib/aws-cloudwatch";
|
|
2
|
-
import type { AutoScalingGroup } from "aws-cdk-lib/aws-autoscaling";
|
|
3
|
-
import type { ITopic } from "aws-cdk-lib/aws-sns";
|
|
4
|
-
import type { ILogGroup } from "aws-cdk-lib/aws-logs";
|
|
5
|
-
import type { Construct } from "constructs";
|
|
6
|
-
export interface ClickHouseAlarmThresholds {
|
|
7
|
-
/** EC2 host CPU % over 5 min. Default 90. */
|
|
8
|
-
cpuThreshold?: number;
|
|
9
|
-
/** EC2 host memory % over 5 min (requires CWAgent). Default 80. */
|
|
10
|
-
memoryThreshold?: number;
|
|
11
|
-
/** EBS root-volume disk % used. Default 70 (warn) — paired with critical at 85. */
|
|
12
|
-
diskWarnThreshold?: number;
|
|
13
|
-
/** EBS root-volume disk % used. Default 85. */
|
|
14
|
-
diskCriticalThreshold?: number;
|
|
15
|
-
}
|
|
16
|
-
export interface ClickHouseAlarmsProps {
|
|
17
|
-
scope: Construct;
|
|
18
|
-
asg: AutoScalingGroup;
|
|
19
|
-
alarmTopic: ITopic;
|
|
20
|
-
/**
|
|
21
|
-
* Webapp log group. Required to wire the stuck-merge alarm — `client.ts`
|
|
22
|
-
* emits `serverLogger.warn("ClickHouse", "Stuck merge detected")` when
|
|
23
|
-
* `system.merges` shows a merge elapsed > 30 min.
|
|
24
|
-
*/
|
|
25
|
-
webappLogGroup: ILogGroup;
|
|
26
|
-
/**
|
|
27
|
-
* Backup-task log group. Required to wire the backup-failure alarm —
|
|
28
|
-
* `BACKUP DATABASE … TO S3(…)` emits `AccessDenied` / `S3Exception` lines
|
|
29
|
-
* when the IAM grant or bucket policy is misconfigured (silent before the
|
|
30
|
-
* alarm landed; the daily backup task exited non-zero with no signal).
|
|
31
|
-
*/
|
|
32
|
-
backupTaskLogGroup: ILogGroup;
|
|
33
|
-
config?: ClickHouseAlarmThresholds;
|
|
34
|
-
}
|
|
35
|
-
/**
|
|
36
|
-
* Single-node ClickHouse posture alarms. Covers host-level CPU + (optional)
|
|
37
|
-
* memory and disk via the CloudWatch Agent metric namespace `CWAgent`, plus
|
|
38
|
-
* two log-driven alarms:
|
|
39
|
-
*
|
|
40
|
-
* - **Stuck merges** — `client.ts` polls `system.merges` every 5 min and logs
|
|
41
|
-
* `serverLogger.warn("ClickHouse", "Stuck merge detected")` when elapsed
|
|
42
|
-
* exceeds 30 min. The metric filter on the webapp log group emits a count
|
|
43
|
-
* metric per match; the alarm fires on Sum >= 1 over 5 min × 2 evaluations.
|
|
44
|
-
* - **Backup failures** — `AccessDenied` or `S3Exception` from the backup
|
|
45
|
-
* task's BACKUP DATABASE TO S3 statement. Closes the silent-failure mode
|
|
46
|
-
* that masked the original IAM-grant misconfiguration (see
|
|
47
|
-
* `designs/2026-04-27-clickhouse-backup-iam-role.md`).
|
|
48
|
-
*/
|
|
49
|
-
export declare function createClickHouseAlarms(props: ClickHouseAlarmsProps): Alarm[];
|