@fjall/components-infrastructure 0.100.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/lambda-assets/cert-generator/asset/index.js +17948 -0
- package/dist/lib/lambda-assets/cert-generator/asset/package.json +4 -0
- package/dist/lib/patterns/aws/clickhouseDatabase.d.ts +49 -1
- package/dist/lib/patterns/aws/clickhouseDatabase.js +137 -20
- package/dist/lib/patterns/aws/clickhouseTls/index.d.ts +1 -0
- package/dist/lib/patterns/aws/clickhouseTls/index.js +1 -0
- package/dist/lib/patterns/aws/clickhouseTls/types.d.ts +48 -0
- package/dist/lib/patterns/aws/computeEcs.d.ts +13 -1
- package/dist/lib/patterns/aws/computeEcs.js +88 -8
- package/dist/lib/patterns/aws/interfaces/database.d.ts +32 -1
- package/dist/lib/patterns/aws/interfaces/database.js +1 -1
- package/dist/lib/resources/aws/database/clickhouseConstants.d.ts +21 -0
- package/dist/lib/resources/aws/database/clickhouseConstants.js +21 -0
- package/dist/lib/resources/aws/database/clickhouseSecurityGroup.d.ts +2 -0
- package/dist/lib/resources/aws/database/clickhouseSecurityGroup.js +2 -0
- package/dist/lib/resources/aws/database/clickhouseUserData.d.ts +21 -0
- package/dist/lib/resources/aws/database/clickhouseUserData.js +48 -3
- package/dist/lib/resources/aws/database/clickhouseXmlRenderer.d.ts +1 -1
- package/dist/lib/resources/aws/database/clickhouseXmlRenderer.js +1 -1
- package/dist/lib/resources/aws/secrets/index.d.ts +2 -0
- package/dist/lib/resources/aws/secrets/index.js +2 -0
- package/dist/lib/resources/aws/secrets/tlsCaSecret.d.ts +13 -0
- package/dist/lib/resources/aws/secrets/tlsCaSecret.js +15 -0
- package/dist/lib/resources/aws/secrets/tlsServerSecret.d.ts +15 -0
- package/dist/lib/resources/aws/secrets/tlsServerSecret.js +17 -0
- package/dist/lib/resources/aws/utilities/index.d.ts +1 -0
- package/dist/lib/resources/aws/utilities/index.js +1 -0
- package/dist/lib/resources/aws/utilities/tlsCertGenerator.d.ts +33 -0
- package/dist/lib/resources/aws/utilities/tlsCertGenerator.js +67 -0
- package/package.json +7 -5
- package/dist/lib/config/aws/__t17fixture.js +0 -3
- package/dist/lib/config/aws/__t17fixtureType.d.ts +0 -2
- package/dist/lib/config/aws/__t17fixtureType.js +0 -1
- package/dist/lib/config/aws/eventBus.d.ts +0 -7
- package/dist/lib/config/aws/eventBus.js +0 -21
- package/dist/lib/config/aws/identityCenterGroupMembership.d.ts +0 -10
- package/dist/lib/config/aws/identityCenterGroupMembership.js +0 -102
- package/dist/lib/config/aws/securityBaseline.d.ts +0 -15
- package/dist/lib/config/aws/securityBaseline.js +0 -27
- package/dist/lib/patterns/aws/_eslint_test_tmp/leak.d.ts +0 -1
- package/dist/lib/patterns/aws/_eslint_test_tmp/leak.js +0 -4
- package/dist/lib/patterns/aws/managedIdentityCenter.d.ts +0 -4
- package/dist/lib/patterns/aws/managedIdentityCenter.js +0 -19
- package/dist/lib/patterns/aws/subdomainHostedZone.d.ts +0 -9
- package/dist/lib/patterns/aws/subdomainHostedZone.js +0 -34
- package/dist/lib/resources/aws/analytics/clickhouse.d.ts +0 -15
- package/dist/lib/resources/aws/analytics/clickhouse.js +0 -310
- package/dist/lib/resources/aws/analytics/clickhouseAlarms.d.ts +0 -49
- package/dist/lib/resources/aws/analytics/clickhouseAlarms.js +0 -140
- package/dist/lib/resources/aws/analytics/clickhouseConstants.d.ts +0 -73
- package/dist/lib/resources/aws/analytics/clickhouseConstants.js +0 -89
- package/dist/lib/resources/aws/analytics/clickhouseSecurityGroup.d.ts +0 -13
- package/dist/lib/resources/aws/analytics/clickhouseSecurityGroup.js +0 -28
- package/dist/lib/resources/aws/analytics/clickhouseTypes.d.ts +0 -59
- package/dist/lib/resources/aws/analytics/clickhouseTypes.js +0 -1
- package/dist/lib/resources/aws/analytics/clickhouseUserData.d.ts +0 -6
- package/dist/lib/resources/aws/analytics/clickhouseUserData.js +0 -299
- package/dist/lib/resources/aws/analytics/index.d.ts +0 -4
- package/dist/lib/resources/aws/analytics/index.js +0 -2
- package/dist/lib/resources/aws/compute/__tmp__/regression-shape.d.ts +0 -2
- package/dist/lib/resources/aws/compute/__tmp__/regression-shape.js +0 -11
- package/dist/lib/resources/aws/messaging/defaultEventBus.d.ts +0 -7
- package/dist/lib/resources/aws/messaging/defaultEventBus.js +0 -21
- package/dist/lib/resources/aws/networking/domain.d.ts +0 -13
- package/dist/lib/resources/aws/networking/domain.js +0 -100
- package/dist/lib/synth_dump.d.ts +0 -1
- package/dist/lib/synth_dump.js +0 -42
- package/dist/lib/utils/bastionFactory.d.ts +0 -10
- package/dist/lib/utils/bastionFactory.js +0 -29
- package/dist/lib/utils/constructMap.d.ts +0 -33
- package/dist/lib/utils/constructMap.js +0 -154
- package/dist/lib/utils/dnsRecords.d.ts +0 -4
- package/dist/lib/utils/dnsRecords.js +0 -104
- /package/dist/lib/{config/aws/__t17fixture.d.ts → patterns/aws/clickhouseTls/types.js} +0 -0
|
@@ -1,140 +0,0 @@
|
|
|
1
|
-
import { Duration } from "aws-cdk-lib";
|
|
2
|
-
import { Alarm, ComparisonOperator, TreatMissingData } from "aws-cdk-lib/aws-cloudwatch";
|
|
3
|
-
import { SnsAction } from "aws-cdk-lib/aws-cloudwatch-actions";
|
|
4
|
-
import { Metric } from "aws-cdk-lib/aws-cloudwatch";
|
|
5
|
-
import { FilterPattern, MetricFilter } from "aws-cdk-lib/aws-logs";
|
|
6
|
-
import { ALARM_DEFAULTS, registerAlarm, buildAlarmDescription } from "../monitoring/alarmDefaults.js";
|
|
7
|
-
const CLICKHOUSE_METRIC_NAMESPACE = "Fjall/ClickHouse";
|
|
8
|
-
/**
|
|
9
|
-
* Single-node ClickHouse posture alarms. Covers host-level CPU + (optional)
|
|
10
|
-
* memory and disk via the CloudWatch Agent metric namespace `CWAgent`, plus
|
|
11
|
-
* two log-driven alarms:
|
|
12
|
-
*
|
|
13
|
-
* - **Stuck merges** — `client.ts` polls `system.merges` every 5 min and logs
|
|
14
|
-
* `serverLogger.warn("ClickHouse", "Stuck merge detected")` when elapsed
|
|
15
|
-
* exceeds 30 min. The metric filter on the webapp log group emits a count
|
|
16
|
-
* metric per match; the alarm fires on Sum >= 1 over 5 min × 2 evaluations.
|
|
17
|
-
* - **Backup failures** — `AccessDenied` or `S3Exception` from the backup
|
|
18
|
-
* task's BACKUP DATABASE TO S3 statement. Closes the silent-failure mode
|
|
19
|
-
* that masked the original IAM-grant misconfiguration (see
|
|
20
|
-
* `designs/2026-04-27-clickhouse-backup-iam-role.md`).
|
|
21
|
-
*/
|
|
22
|
-
export function createClickHouseAlarms(props) {
|
|
23
|
-
const { scope, asg, alarmTopic, webappLogGroup, backupTaskLogGroup, config = {} } = props;
|
|
24
|
-
const alarms = [];
|
|
25
|
-
const snsAction = new SnsAction(alarmTopic);
|
|
26
|
-
const asgName = asg.autoScalingGroupName;
|
|
27
|
-
const cpuAlarm = new Alarm(scope, "ClickHouseCpuAlarm", {
|
|
28
|
-
alarmDescription: buildAlarmDescription("ClickHouse host CPU utilisation exceeds threshold", undefined),
|
|
29
|
-
metric: new Metric({
|
|
30
|
-
namespace: "AWS/EC2",
|
|
31
|
-
metricName: "CPUUtilization",
|
|
32
|
-
dimensionsMap: { AutoScalingGroupName: asgName },
|
|
33
|
-
period: ALARM_DEFAULTS.EVALUATION_PERIOD,
|
|
34
|
-
statistic: "Average"
|
|
35
|
-
}),
|
|
36
|
-
threshold: config.cpuThreshold ?? 90,
|
|
37
|
-
evaluationPeriods: 3,
|
|
38
|
-
datapointsToAlarm: 2,
|
|
39
|
-
comparisonOperator: ComparisonOperator.GREATER_THAN_THRESHOLD,
|
|
40
|
-
treatMissingData: TreatMissingData.NOT_BREACHING
|
|
41
|
-
});
|
|
42
|
-
registerAlarm(cpuAlarm, snsAction, alarms);
|
|
43
|
-
const memoryAlarm = new Alarm(scope, "ClickHouseMemoryAlarm", {
|
|
44
|
-
alarmDescription: buildAlarmDescription("ClickHouse host memory utilisation exceeds threshold (CWAgent)", undefined),
|
|
45
|
-
metric: new Metric({
|
|
46
|
-
namespace: "CWAgent",
|
|
47
|
-
metricName: "mem_used_percent",
|
|
48
|
-
dimensionsMap: { AutoScalingGroupName: asgName },
|
|
49
|
-
period: ALARM_DEFAULTS.EVALUATION_PERIOD,
|
|
50
|
-
statistic: "Average"
|
|
51
|
-
}),
|
|
52
|
-
threshold: config.memoryThreshold ?? 80,
|
|
53
|
-
evaluationPeriods: 3,
|
|
54
|
-
datapointsToAlarm: 2,
|
|
55
|
-
comparisonOperator: ComparisonOperator.GREATER_THAN_THRESHOLD,
|
|
56
|
-
treatMissingData: TreatMissingData.NOT_BREACHING
|
|
57
|
-
});
|
|
58
|
-
registerAlarm(memoryAlarm, snsAction, alarms);
|
|
59
|
-
const diskWarnAlarm = new Alarm(scope, "ClickHouseDiskWarnAlarm", {
|
|
60
|
-
alarmDescription: buildAlarmDescription("ClickHouse data volume above 70% used — plan growth response", undefined),
|
|
61
|
-
metric: new Metric({
|
|
62
|
-
namespace: "CWAgent",
|
|
63
|
-
metricName: "disk_used_percent",
|
|
64
|
-
dimensionsMap: { AutoScalingGroupName: asgName },
|
|
65
|
-
period: Duration.minutes(15),
|
|
66
|
-
statistic: "Average"
|
|
67
|
-
}),
|
|
68
|
-
threshold: config.diskWarnThreshold ?? 70,
|
|
69
|
-
evaluationPeriods: 2,
|
|
70
|
-
datapointsToAlarm: 2,
|
|
71
|
-
comparisonOperator: ComparisonOperator.GREATER_THAN_THRESHOLD,
|
|
72
|
-
treatMissingData: TreatMissingData.NOT_BREACHING
|
|
73
|
-
});
|
|
74
|
-
registerAlarm(diskWarnAlarm, snsAction, alarms);
|
|
75
|
-
const diskCriticalAlarm = new Alarm(scope, "ClickHouseDiskCriticalAlarm", {
|
|
76
|
-
alarmDescription: buildAlarmDescription("ClickHouse data volume above 85% used — imminent insert failures", undefined),
|
|
77
|
-
metric: new Metric({
|
|
78
|
-
namespace: "CWAgent",
|
|
79
|
-
metricName: "disk_used_percent",
|
|
80
|
-
dimensionsMap: { AutoScalingGroupName: asgName },
|
|
81
|
-
period: Duration.minutes(5),
|
|
82
|
-
statistic: "Average"
|
|
83
|
-
}),
|
|
84
|
-
threshold: config.diskCriticalThreshold ?? 85,
|
|
85
|
-
evaluationPeriods: 2,
|
|
86
|
-
datapointsToAlarm: 2,
|
|
87
|
-
comparisonOperator: ComparisonOperator.GREATER_THAN_THRESHOLD,
|
|
88
|
-
treatMissingData: TreatMissingData.NOT_BREACHING
|
|
89
|
-
});
|
|
90
|
-
registerAlarm(diskCriticalAlarm, snsAction, alarms);
|
|
91
|
-
const stuckMergeMetricName = "ClickHouseStuckMergeCount";
|
|
92
|
-
new MetricFilter(scope, "ClickHouseStuckMergeMetricFilter", {
|
|
93
|
-
logGroup: webappLogGroup,
|
|
94
|
-
metricNamespace: CLICKHOUSE_METRIC_NAMESPACE,
|
|
95
|
-
metricName: stuckMergeMetricName,
|
|
96
|
-
filterPattern: FilterPattern.literal('"Stuck merge detected"'),
|
|
97
|
-
metricValue: "1",
|
|
98
|
-
defaultValue: 0
|
|
99
|
-
});
|
|
100
|
-
const stuckMergeAlarm = new Alarm(scope, "ClickHouseStuckMergeAlarm", {
|
|
101
|
-
alarmDescription: buildAlarmDescription("ClickHouse merge stuck > 30 min — investigate parts pressure or replica health", undefined),
|
|
102
|
-
metric: new Metric({
|
|
103
|
-
namespace: CLICKHOUSE_METRIC_NAMESPACE,
|
|
104
|
-
metricName: stuckMergeMetricName,
|
|
105
|
-
period: Duration.minutes(5),
|
|
106
|
-
statistic: "Sum"
|
|
107
|
-
}),
|
|
108
|
-
threshold: 1,
|
|
109
|
-
evaluationPeriods: 2,
|
|
110
|
-
datapointsToAlarm: 2,
|
|
111
|
-
comparisonOperator: ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
|
|
112
|
-
treatMissingData: TreatMissingData.NOT_BREACHING
|
|
113
|
-
});
|
|
114
|
-
registerAlarm(stuckMergeAlarm, snsAction, alarms);
|
|
115
|
-
const backupFailureMetricName = "ClickHouseBackupFailureCount";
|
|
116
|
-
new MetricFilter(scope, "ClickHouseBackupFailureMetricFilter", {
|
|
117
|
-
logGroup: backupTaskLogGroup,
|
|
118
|
-
metricNamespace: CLICKHOUSE_METRIC_NAMESPACE,
|
|
119
|
-
metricName: backupFailureMetricName,
|
|
120
|
-
filterPattern: FilterPattern.anyTerm("AccessDenied", "S3Exception"),
|
|
121
|
-
metricValue: "1",
|
|
122
|
-
defaultValue: 0
|
|
123
|
-
});
|
|
124
|
-
const backupFailureAlarm = new Alarm(scope, "ClickHouseBackupFailureAlarm", {
|
|
125
|
-
alarmDescription: buildAlarmDescription("ClickHouse BACKUP TO S3 emitted AccessDenied/S3Exception — verify ASG instance role grant on backup bucket", undefined),
|
|
126
|
-
metric: new Metric({
|
|
127
|
-
namespace: CLICKHOUSE_METRIC_NAMESPACE,
|
|
128
|
-
metricName: backupFailureMetricName,
|
|
129
|
-
period: Duration.hours(1),
|
|
130
|
-
statistic: "Sum"
|
|
131
|
-
}),
|
|
132
|
-
threshold: 1,
|
|
133
|
-
evaluationPeriods: 1,
|
|
134
|
-
datapointsToAlarm: 1,
|
|
135
|
-
comparisonOperator: ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
|
|
136
|
-
treatMissingData: TreatMissingData.NOT_BREACHING
|
|
137
|
-
});
|
|
138
|
-
registerAlarm(backupFailureAlarm, snsAction, alarms);
|
|
139
|
-
return alarms;
|
|
140
|
-
}
|
|
@@ -1,73 +0,0 @@
|
|
|
1
|
-
/** Cluster/task family name used for ECS resources. */
|
|
2
|
-
export declare const CLICKHOUSE_CLUSTER_NAME = "clickhouse-analytics";
|
|
3
|
-
/** Default EC2 instance type for ClickHouse (Graviton — best cost/performance). */
|
|
4
|
-
export declare const DEFAULT_CLICKHOUSE_INSTANCE_TYPE = "t4g.medium";
|
|
5
|
-
/** ClickHouse container image. */
|
|
6
|
-
export declare const CLICKHOUSE_IMAGE = "clickhouse/clickhouse-server:26.3-alpine";
|
|
7
|
-
/** EBS volume configuration. */
|
|
8
|
-
export declare const CLICKHOUSE_EBS_VOLUME_SIZE_GB = 80;
|
|
9
|
-
export declare const CLICKHOUSE_EBS_IOPS = 3000;
|
|
10
|
-
export declare const CLICKHOUSE_EBS_THROUGHPUT_MBPS = 125;
|
|
11
|
-
/** ECS task resource allocation (t4g.medium = 4 GB total). */
|
|
12
|
-
export declare const CLICKHOUSE_TASK_MEMORY_MIB = 3072;
|
|
13
|
-
export declare const CLICKHOUSE_TASK_CPU_UNITS = 1024;
|
|
14
|
-
/** ClickHouse ports. */
|
|
15
|
-
export declare const CLICKHOUSE_HTTP_PORT = 8123;
|
|
16
|
-
export declare const CLICKHOUSE_NATIVE_PORT = 9000;
|
|
17
|
-
export declare const CLICKHOUSE_PROMETHEUS_PORT = 9363;
|
|
18
|
-
/** EBS device name for the data volume (must match user data script). */
|
|
19
|
-
export declare const CLICKHOUSE_EBS_DEVICE_NAME = "/dev/xvdf";
|
|
20
|
-
/** EBS mount path on the EC2 host. */
|
|
21
|
-
export declare const CLICKHOUSE_DATA_MOUNT_PATH = "/mnt/clickhouse-data";
|
|
22
|
-
/** Secrets Manager path prefix. */
|
|
23
|
-
export declare const CLICKHOUSE_SECRETS_PREFIX = "fjall/clickhouse";
|
|
24
|
-
/** Secret names (under the prefix). */
|
|
25
|
-
export declare const CLICKHOUSE_SECRET_NAMES: {
|
|
26
|
-
readonly APP_PASSWORD: "app-password";
|
|
27
|
-
readonly AUDIT_PASSWORD: "audit-password";
|
|
28
|
-
readonly BACKUP_PASSWORD: "backup-password";
|
|
29
|
-
readonly SCHEMA_PASSWORD: "schema-password";
|
|
30
|
-
};
|
|
31
|
-
/** Shared secret generation options (all ClickHouse users share the same policy). */
|
|
32
|
-
export declare const CLICKHOUSE_SECRET_OPTIONS: {
|
|
33
|
-
readonly excludePunctuation: true;
|
|
34
|
-
readonly passwordLength: 32;
|
|
35
|
-
};
|
|
36
|
-
/** Health check configuration. */
|
|
37
|
-
export declare const CLICKHOUSE_HEALTH_CHECK: {
|
|
38
|
-
readonly INTERVAL_SECONDS: 30;
|
|
39
|
-
readonly TIMEOUT_SECONDS: 5;
|
|
40
|
-
readonly RETRIES: 3;
|
|
41
|
-
readonly START_PERIOD_SECONDS: 60;
|
|
42
|
-
};
|
|
43
|
-
/** OPTIMIZE TABLE FINAL schedule.
|
|
44
|
-
* RMT tables carry min_age_to_force_merge_seconds=600 so the engine already merges
|
|
45
|
-
* old parts within 10 min; this task is a safety net for MVs (no engine-level setting)
|
|
46
|
-
* and for ReplacingMergeTree dedup under skewed write patterns. 6 hours is sufficient. */
|
|
47
|
-
export declare const OPTIMISE_FINAL_SCHEDULE = "rate(6 hours)";
|
|
48
|
-
/** Tables requiring periodic OPTIMIZE FINAL (ReplacingMergeTree only).
|
|
49
|
-
* Keep in sync with REPLACING_MERGE_TREE_TABLES in
|
|
50
|
-
* webapp/app/.server/lib/clickhouse/tenantQuery.ts (auto-FINAL). */
|
|
51
|
-
export declare const REPLACING_MERGE_TREE_TABLES: readonly ["application_metrics", "cost_records", "log_fingerprints", "insights", "asset_inventory"];
|
|
52
|
-
/** Subdirectory on the EBS volume for server config files (must match CDK volume mount). */
|
|
53
|
-
export declare const CLICKHOUSE_CONFIG_SUBDIR = "server-config.d";
|
|
54
|
-
/** Subdirectory on the EBS volume for users config files (must match CDK volume mount). */
|
|
55
|
-
export declare const CLICKHOUSE_USERS_SUBDIR = "server-users.d";
|
|
56
|
-
/** Cloud Map namespace for ClickHouse service discovery. */
|
|
57
|
-
export declare const CLICKHOUSE_CLOUDMAP_NAMESPACE = "clickhouse.local";
|
|
58
|
-
/** Cloud Map service name (resolves to clickhouse.clickhouse.local). */
|
|
59
|
-
export declare const CLICKHOUSE_CLOUDMAP_SERVICE_NAME = "clickhouse";
|
|
60
|
-
/** Materialised views that benefit from periodic OPTIMIZE to reduce part count at read time.
|
|
61
|
-
* These are not ReplacingMergeTree (no dedup needed) but un-merged parts force
|
|
62
|
-
* read-time aggregation which degrades query performance. */
|
|
63
|
-
export declare const OPTIMISE_MV_TABLES: readonly ["metrics_hourly_mv", "metrics_daily_mv", "response_time_quantiles_hourly_mv", "deployment_duration_quantiles_daily_mv", "log_severity_hourly_mv", "compliance_score_daily_mv", "ai_usage_daily_mv", "finding_daily_aggregate", "insight_pattern_dismissals"];
|
|
64
|
-
/** Resource allocation for the lightweight optimise task. */
|
|
65
|
-
export declare const OPTIMISE_TASK_MEMORY_MIB = 256;
|
|
66
|
-
export declare const OPTIMISE_TASK_CPU_UNITS = 256;
|
|
67
|
-
/** Automated backup schedule (daily 03:00 UTC — low-traffic window). */
|
|
68
|
-
export declare const BACKUP_SCHEDULE = "cron(0 3 * * ? *)";
|
|
69
|
-
/** Resource allocation for the backup task (lightweight — clickhouse-client only). */
|
|
70
|
-
export declare const BACKUP_TASK_MEMORY_MIB = 256;
|
|
71
|
-
export declare const BACKUP_TASK_CPU_UNITS = 256;
|
|
72
|
-
/** Backup object expiration: 14 days (retains 14 daily snapshots). */
|
|
73
|
-
export declare const BACKUP_RETENTION_DAYS = 14;
|
|
@@ -1,89 +0,0 @@
|
|
|
1
|
-
/** Cluster/task family name used for ECS resources. */
|
|
2
|
-
export const CLICKHOUSE_CLUSTER_NAME = "clickhouse-analytics";
|
|
3
|
-
/** Default EC2 instance type for ClickHouse (Graviton — best cost/performance). */
|
|
4
|
-
export const DEFAULT_CLICKHOUSE_INSTANCE_TYPE = "t4g.medium";
|
|
5
|
-
/** ClickHouse container image. */
|
|
6
|
-
export const CLICKHOUSE_IMAGE = "clickhouse/clickhouse-server:26.3-alpine";
|
|
7
|
-
/** EBS volume configuration. */
|
|
8
|
-
export const CLICKHOUSE_EBS_VOLUME_SIZE_GB = 80;
|
|
9
|
-
export const CLICKHOUSE_EBS_IOPS = 3000;
|
|
10
|
-
export const CLICKHOUSE_EBS_THROUGHPUT_MBPS = 125;
|
|
11
|
-
/** ECS task resource allocation (t4g.medium = 4 GB total). */
|
|
12
|
-
export const CLICKHOUSE_TASK_MEMORY_MIB = 3072;
|
|
13
|
-
export const CLICKHOUSE_TASK_CPU_UNITS = 1024;
|
|
14
|
-
/** ClickHouse ports. */
|
|
15
|
-
export const CLICKHOUSE_HTTP_PORT = 8123;
|
|
16
|
-
export const CLICKHOUSE_NATIVE_PORT = 9000;
|
|
17
|
-
export const CLICKHOUSE_PROMETHEUS_PORT = 9363;
|
|
18
|
-
/** EBS device name for the data volume (must match user data script). */
|
|
19
|
-
export const CLICKHOUSE_EBS_DEVICE_NAME = "/dev/xvdf";
|
|
20
|
-
/** EBS mount path on the EC2 host. */
|
|
21
|
-
export const CLICKHOUSE_DATA_MOUNT_PATH = "/mnt/clickhouse-data";
|
|
22
|
-
/** Secrets Manager path prefix. */
|
|
23
|
-
export const CLICKHOUSE_SECRETS_PREFIX = "fjall/clickhouse";
|
|
24
|
-
/** Secret names (under the prefix). */
|
|
25
|
-
export const CLICKHOUSE_SECRET_NAMES = {
|
|
26
|
-
APP_PASSWORD: "app-password",
|
|
27
|
-
AUDIT_PASSWORD: "audit-password",
|
|
28
|
-
BACKUP_PASSWORD: "backup-password",
|
|
29
|
-
SCHEMA_PASSWORD: "schema-password"
|
|
30
|
-
};
|
|
31
|
-
/** Shared secret generation options (all ClickHouse users share the same policy). */
|
|
32
|
-
export const CLICKHOUSE_SECRET_OPTIONS = {
|
|
33
|
-
excludePunctuation: true,
|
|
34
|
-
passwordLength: 32
|
|
35
|
-
};
|
|
36
|
-
/** Health check configuration. */
|
|
37
|
-
export const CLICKHOUSE_HEALTH_CHECK = {
|
|
38
|
-
INTERVAL_SECONDS: 30,
|
|
39
|
-
TIMEOUT_SECONDS: 5,
|
|
40
|
-
RETRIES: 3,
|
|
41
|
-
START_PERIOD_SECONDS: 60
|
|
42
|
-
};
|
|
43
|
-
/** OPTIMIZE TABLE FINAL schedule.
|
|
44
|
-
* RMT tables carry min_age_to_force_merge_seconds=600 so the engine already merges
|
|
45
|
-
* old parts within 10 min; this task is a safety net for MVs (no engine-level setting)
|
|
46
|
-
* and for ReplacingMergeTree dedup under skewed write patterns. 6 hours is sufficient. */
|
|
47
|
-
export const OPTIMISE_FINAL_SCHEDULE = "rate(6 hours)";
|
|
48
|
-
/** Tables requiring periodic OPTIMIZE FINAL (ReplacingMergeTree only).
|
|
49
|
-
* Keep in sync with REPLACING_MERGE_TREE_TABLES in
|
|
50
|
-
* webapp/app/.server/lib/clickhouse/tenantQuery.ts (auto-FINAL). */
|
|
51
|
-
export const REPLACING_MERGE_TREE_TABLES = [
|
|
52
|
-
"application_metrics",
|
|
53
|
-
"cost_records",
|
|
54
|
-
"log_fingerprints",
|
|
55
|
-
"insights",
|
|
56
|
-
"asset_inventory"
|
|
57
|
-
];
|
|
58
|
-
/** Subdirectory on the EBS volume for server config files (must match CDK volume mount). */
|
|
59
|
-
export const CLICKHOUSE_CONFIG_SUBDIR = "server-config.d";
|
|
60
|
-
/** Subdirectory on the EBS volume for users config files (must match CDK volume mount). */
|
|
61
|
-
export const CLICKHOUSE_USERS_SUBDIR = "server-users.d";
|
|
62
|
-
/** Cloud Map namespace for ClickHouse service discovery. */
|
|
63
|
-
export const CLICKHOUSE_CLOUDMAP_NAMESPACE = "clickhouse.local";
|
|
64
|
-
/** Cloud Map service name (resolves to clickhouse.clickhouse.local). */
|
|
65
|
-
export const CLICKHOUSE_CLOUDMAP_SERVICE_NAME = "clickhouse";
|
|
66
|
-
/** Materialised views that benefit from periodic OPTIMIZE to reduce part count at read time.
|
|
67
|
-
* These are not ReplacingMergeTree (no dedup needed) but un-merged parts force
|
|
68
|
-
* read-time aggregation which degrades query performance. */
|
|
69
|
-
export const OPTIMISE_MV_TABLES = [
|
|
70
|
-
"metrics_hourly_mv",
|
|
71
|
-
"metrics_daily_mv",
|
|
72
|
-
"response_time_quantiles_hourly_mv",
|
|
73
|
-
"deployment_duration_quantiles_daily_mv",
|
|
74
|
-
"log_severity_hourly_mv",
|
|
75
|
-
"compliance_score_daily_mv",
|
|
76
|
-
"ai_usage_daily_mv",
|
|
77
|
-
"finding_daily_aggregate",
|
|
78
|
-
"insight_pattern_dismissals"
|
|
79
|
-
];
|
|
80
|
-
/** Resource allocation for the lightweight optimise task. */
|
|
81
|
-
export const OPTIMISE_TASK_MEMORY_MIB = 256;
|
|
82
|
-
export const OPTIMISE_TASK_CPU_UNITS = 256;
|
|
83
|
-
/** Automated backup schedule (daily 03:00 UTC — low-traffic window). */
|
|
84
|
-
export const BACKUP_SCHEDULE = "cron(0 3 * * ? *)";
|
|
85
|
-
/** Resource allocation for the backup task (lightweight — clickhouse-client only). */
|
|
86
|
-
export const BACKUP_TASK_MEMORY_MIB = 256;
|
|
87
|
-
export const BACKUP_TASK_CPU_UNITS = 256;
|
|
88
|
-
/** Backup object expiration: 14 days (retains 14 daily snapshots). */
|
|
89
|
-
export const BACKUP_RETENTION_DAYS = 14;
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
import { type ISecurityGroup, type IVpc } from "aws-cdk-lib/aws-ec2";
|
|
2
|
-
import type { Construct } from "constructs";
|
|
3
|
-
import { SecurityGroup } from "../networking/securityGroup.js";
|
|
4
|
-
/**
|
|
5
|
-
* Creates the ClickHouse security group.
|
|
6
|
-
*
|
|
7
|
-
* Inbound:
|
|
8
|
-
* - TCP 8123 from webapp ECS service SG (HTTP queries)
|
|
9
|
-
*
|
|
10
|
-
* Outbound:
|
|
11
|
-
* - HTTPS 443 to 0.0.0.0/0 (R2 endpoint + Secrets Manager)
|
|
12
|
-
*/
|
|
13
|
-
export declare function createClickHouseSecurityGroup(scope: Construct, vpc: IVpc, webappSecurityGroup: ISecurityGroup): SecurityGroup;
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
import { Peer, Port } from "aws-cdk-lib/aws-ec2";
|
|
2
|
-
import { SecurityGroup } from "../networking/securityGroup.js";
|
|
3
|
-
import { CLICKHOUSE_HTTP_PORT, CLICKHOUSE_NATIVE_PORT } from "./clickhouseConstants.js";
|
|
4
|
-
/**
|
|
5
|
-
* Creates the ClickHouse security group.
|
|
6
|
-
*
|
|
7
|
-
* Inbound:
|
|
8
|
-
* - TCP 8123 from webapp ECS service SG (HTTP queries)
|
|
9
|
-
*
|
|
10
|
-
* Outbound:
|
|
11
|
-
* - HTTPS 443 to 0.0.0.0/0 (R2 endpoint + Secrets Manager)
|
|
12
|
-
*/
|
|
13
|
-
export function createClickHouseSecurityGroup(scope, vpc, webappSecurityGroup) {
|
|
14
|
-
const sg = new SecurityGroup(scope, "ClickHouseSecurityGroup", {
|
|
15
|
-
vpc,
|
|
16
|
-
description: "Security group for ClickHouse analytics instance",
|
|
17
|
-
allowAllOutbound: false
|
|
18
|
-
});
|
|
19
|
-
// Inbound: HTTP API from webapp
|
|
20
|
-
sg.addIngressRule(webappSecurityGroup, Port.tcp(CLICKHOUSE_HTTP_PORT), "ClickHouse HTTP API from webapp ECS service");
|
|
21
|
-
// Inbound: Native protocol from optimise scheduled task (same SG, self-referencing)
|
|
22
|
-
sg.addIngressRule(sg, Port.tcp(CLICKHOUSE_NATIVE_PORT), "ClickHouse native protocol from optimise task");
|
|
23
|
-
// Outbound: HTTPS for R2 cold storage and Secrets Manager
|
|
24
|
-
sg.addEgressRule(Peer.anyIpv4(), Port.tcp(443), "HTTPS to R2 and Secrets Manager endpoints");
|
|
25
|
-
// Outbound: Native protocol to ClickHouse (optimise task connection)
|
|
26
|
-
sg.addEgressRule(sg, Port.tcp(CLICKHOUSE_NATIVE_PORT), "ClickHouse native protocol for optimise task");
|
|
27
|
-
return sg;
|
|
28
|
-
}
|
|
@@ -1,59 +0,0 @@
|
|
|
1
|
-
import type { IVpc, ISecurityGroup } from "aws-cdk-lib/aws-ec2";
|
|
2
|
-
import type { ILogGroup } from "aws-cdk-lib/aws-logs";
|
|
3
|
-
import type { IBucket } from "aws-cdk-lib/aws-s3";
|
|
4
|
-
import type { ISecret } from "aws-cdk-lib/aws-secretsmanager";
|
|
5
|
-
import type { ITopic } from "aws-cdk-lib/aws-sns";
|
|
6
|
-
/** Props for the ClickHouse CDK construct. */
|
|
7
|
-
export interface ClickHouseProps {
|
|
8
|
-
/** VPC to deploy into. */
|
|
9
|
-
vpc: IVpc;
|
|
10
|
-
/**
|
|
11
|
-
* EC2 instance type for ClickHouse.
|
|
12
|
-
* Overridden by CDK context parameter `clickhouseInstanceType` if set.
|
|
13
|
-
* Default: t4g.medium (4 GB RAM).
|
|
14
|
-
*/
|
|
15
|
-
instanceType?: string;
|
|
16
|
-
/**
|
|
17
|
-
* Security group of the webapp ECS service.
|
|
18
|
-
* Used to allow inbound HTTP (8123) from the webapp.
|
|
19
|
-
*/
|
|
20
|
-
webappSecurityGroup: ISecurityGroup;
|
|
21
|
-
/**
|
|
22
|
-
* R2 configuration for cold storage.
|
|
23
|
-
* If omitted, tiered storage is disabled (local-only).
|
|
24
|
-
*/
|
|
25
|
-
r2Config?: ClickHouseR2Config;
|
|
26
|
-
/**
|
|
27
|
-
* SNS topic for CloudWatch alarms (CPU, memory, disk, stuck merges).
|
|
28
|
-
* If omitted, posture alarms are not created.
|
|
29
|
-
*/
|
|
30
|
-
alarmTopic?: ITopic;
|
|
31
|
-
/**
|
|
32
|
-
* Webapp log group, required when `alarmTopic` is set so the stuck-merge
|
|
33
|
-
* metric filter can read the structured warning emitted by `client.ts`.
|
|
34
|
-
*/
|
|
35
|
-
webappLogGroup?: ILogGroup;
|
|
36
|
-
}
|
|
37
|
-
/** Cloudflare R2 configuration for tiered storage and backups. */
|
|
38
|
-
export interface ClickHouseR2Config {
|
|
39
|
-
/** Cloudflare account ID. */
|
|
40
|
-
accountId: string;
|
|
41
|
-
/** R2 access key (stored in Secrets Manager). */
|
|
42
|
-
accessKeySecret: ISecret;
|
|
43
|
-
/** R2 secret key (stored in Secrets Manager). */
|
|
44
|
-
secretKeySecret: ISecret;
|
|
45
|
-
}
|
|
46
|
-
/** Outputs from the ClickHouse construct for use by other constructs. */
|
|
47
|
-
export interface ClickHouseOutputs {
|
|
48
|
-
/** Security group for the ClickHouse instance (for connection rules). */
|
|
49
|
-
securityGroup: ISecurityGroup;
|
|
50
|
-
/** S3 bucket for weekly automated backups (for restore operations). */
|
|
51
|
-
backupBucket: IBucket;
|
|
52
|
-
/** Secrets Manager secrets for ClickHouse passwords. */
|
|
53
|
-
secrets: {
|
|
54
|
-
appPassword: ISecret;
|
|
55
|
-
auditPassword: ISecret;
|
|
56
|
-
backupPassword: ISecret;
|
|
57
|
-
schemaPassword: ISecret;
|
|
58
|
-
};
|
|
59
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
export interface ClickHouseUserDataOptions {
|
|
2
|
-
/** Cloudflare account ID for R2 cold storage. If omitted, local-only storage is used. */
|
|
3
|
-
cfAccountId?: string;
|
|
4
|
-
}
|
|
5
|
-
export declare const USERS_CONFIG_XML = "<clickhouse>\n <users>\n <default>\n <networks>\n <ip>127.0.0.1</ip>\n <ip>::1</ip>\n </networks>\n </default>\n </users>\n <profiles>\n <default>\n <optimize_move_to_prewhere>1</optimize_move_to_prewhere>\n <!-- ALTER TABLE ... MODIFY TTL on a 30-day-partitioned table would otherwise\n trigger an immediate full-table rewrite (default = 1). On the t4g.medium\n box that's a merge-pool starvation event. Keep TTL changes lazy: parts\n re-evaluate TTL on their next natural merge, no forced rewrite. -->\n <materialize_ttl_after_modify>0</materialize_ttl_after_modify>\n </default>\n <app_writer>\n <max_threads>2</max_threads>\n <max_insert_threads>1</max_insert_threads>\n <max_concurrent_queries_for_user>4</max_concurrent_queries_for_user>\n <log_queries_min_query_duration_ms>100</log_queries_min_query_duration_ms>\n <optimize_move_to_prewhere>1</optimize_move_to_prewhere>\n <use_query_condition_cache>1</use_query_condition_cache>\n <!-- Re-enable skip indexes under FINAL (tenantQuery auto-FINALs RMT tables;\n default disables idx_aws_account, idx_application, idx_dedup, idx_fingerprint). -->\n <use_skip_indexes_if_final>1</use_skip_indexes_if_final>\n <!-- Tenant-isolation guards (ClickHouse PR #91065 fix). Belt-and-braces with the\n per-user SQL SETTINGS in 002-users.sql \u2014 keep both so `CREATE OR REPLACE USER`\n cannot regress this. Without these flags, FINAL queries on un-merged\n ReplacingMergeTree parts can leak across tenants. -->\n <apply_row_policy_after_final>1</apply_row_policy_after_final>\n <apply_prewhere_after_final>1</apply_prewhere_after_final>\n <do_not_merge_across_partitions_select_final>1</do_not_merge_across_partitions_select_final>\n <async_insert>1</async_insert>\n <wait_for_async_insert>1</wait_for_async_insert>\n <async_insert_max_data_size>10000000</async_insert_max_data_size>\n <!-- Adaptive batching: tune flush window between 50 ms (low-latency rare inserts)\n and 2 s (absorbs bursts). A single fixed value is silently overridden by the\n adaptive algorithm. -->\n <async_insert_busy_timeout_min_ms>50</async_insert_busy_timeout_min_ms>\n <async_insert_busy_timeout_max_ms>2000</async_insert_busy_timeout_max_ms>\n <async_insert_use_adaptive_busy_timeout>1</async_insert_use_adaptive_busy_timeout>\n <!-- Server-side deduplication of async inserts. Latent retry safety net:\n if a producer retries the same insert window (network hiccup, lambda re-run,\n SQS redelivery), the second attempt collapses against the first. As of CH 26.1\n this also propagates end-to-end through dependent materialised views \u2014 without\n it, a retried insert could double-count in metrics_hourly_mv / log_severity_hourly_mv\n even if the base table dedups. CH pin is 26.3 so the propagation fix is in. -->\n <async_insert_deduplicate>1</async_insert_deduplicate>\n <input_format_parallel_parsing>0</input_format_parallel_parsing>\n <output_format_parallel_formatting>0</output_format_parallel_formatting>\n <!-- Lazy materialisation (CH 25.4+): for `SELECT * ... LIMIT N` shapes the planner\n reads only the columns needed to evaluate ORDER BY / WHERE, then fetches the\n remaining columns for the surviving N rows. Order-of-magnitude I/O reduction\n on dashboard queries (e.g. getLatestMetrics LIMIT 1 BY application_id). -->\n <query_plan_optimize_lazy_materialization>1</query_plan_optimize_lazy_materialization>\n <!-- Per-query memory cap (overrides server-wide max_memory_usage of 1 GB\n to give app_writer 2 GB headroom). Belt-and-braces with the inline\n SETTINGS in 002-users.sql so neither layer can drift alone. -->\n <max_memory_usage>2000000000</max_memory_usage>\n <max_memory_usage_for_user>2684354560</max_memory_usage_for_user>\n <max_bytes_before_external_sort>536870912</max_bytes_before_external_sort>\n <max_bytes_before_external_group_by>536870912</max_bytes_before_external_group_by>\n <!-- Per-query caps. Belt-and-braces with the inline SETTINGS in\n 002-users.sql so `CREATE OR REPLACE USER` cannot regress the bound. -->\n <max_execution_time>30</max_execution_time>\n <max_rows_to_read>10000000</max_rows_to_read>\n </app_writer>\n <audit_writer>\n <max_threads>1</max_threads>\n <max_insert_threads>1</max_insert_threads>\n <max_concurrent_queries_for_user>2</max_concurrent_queries_for_user>\n <max_memory_usage>500000000</max_memory_usage>\n <max_execution_time>10</max_execution_time>\n <async_insert>1</async_insert>\n <wait_for_async_insert>1</wait_for_async_insert>\n </audit_writer>\n <backup_reader>\n <max_threads>2</max_threads>\n <max_concurrent_queries_for_user>1</max_concurrent_queries_for_user>\n <max_memory_usage>1000000000</max_memory_usage>\n <max_execution_time>3600</max_execution_time>\n </backup_reader>\n <schema_admin>\n <max_threads>2</max_threads>\n <max_concurrent_queries_for_user>1</max_concurrent_queries_for_user>\n <max_memory_usage>1000000000</max_memory_usage>\n <max_execution_time>1800</max_execution_time>\n </schema_admin>\n </profiles>\n <quotas>\n <tenant_default>\n <interval>\n <duration>3600</duration>\n <queries>1000</queries>\n <result_rows>10000000</result_rows>\n </interval>\n </tenant_default>\n </quotas>\n</clickhouse>";
|
|
6
|
-
export declare function generateClickHouseUserData(options?: ClickHouseUserDataOptions): string;
|