@fjall/components-infrastructure 0.102.0 → 2.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/dist/lib/lambda-assets/cert-generator/asset/index.js +17948 -0
  2. package/dist/lib/lambda-assets/cert-generator/asset/package.json +4 -0
  3. package/dist/lib/patterns/aws/clickhouseDatabase.d.ts +45 -2
  4. package/dist/lib/patterns/aws/clickhouseDatabase.js +156 -32
  5. package/dist/lib/patterns/aws/clickhouseTls/index.d.ts +1 -0
  6. package/dist/lib/patterns/aws/clickhouseTls/index.js +1 -0
  7. package/dist/lib/patterns/aws/clickhouseTls/types.d.ts +48 -0
  8. package/dist/lib/patterns/aws/interfaces/database.d.ts +7 -5
  9. package/dist/lib/resources/aws/database/clickhouseConstants.d.ts +21 -0
  10. package/dist/lib/resources/aws/database/clickhouseConstants.js +21 -0
  11. package/dist/lib/resources/aws/database/clickhouseSecurityGroup.d.ts +2 -0
  12. package/dist/lib/resources/aws/database/clickhouseSecurityGroup.js +2 -0
  13. package/dist/lib/resources/aws/database/clickhouseUserData.d.ts +21 -0
  14. package/dist/lib/resources/aws/database/clickhouseUserData.js +68 -3
  15. package/dist/lib/resources/aws/database/clickhouseXmlRenderer.d.ts +1 -1
  16. package/dist/lib/resources/aws/database/clickhouseXmlRenderer.js +1 -1
  17. package/dist/lib/resources/aws/secrets/index.d.ts +2 -0
  18. package/dist/lib/resources/aws/secrets/index.js +2 -0
  19. package/dist/lib/resources/aws/secrets/tlsCaSecret.d.ts +13 -0
  20. package/dist/lib/resources/aws/secrets/tlsCaSecret.js +15 -0
  21. package/dist/lib/resources/aws/secrets/tlsServerSecret.d.ts +15 -0
  22. package/dist/lib/resources/aws/secrets/tlsServerSecret.js +17 -0
  23. package/dist/lib/resources/aws/utilities/index.d.ts +1 -0
  24. package/dist/lib/resources/aws/utilities/index.js +1 -0
  25. package/dist/lib/resources/aws/utilities/tlsCertGenerator.d.ts +33 -0
  26. package/dist/lib/resources/aws/utilities/tlsCertGenerator.js +67 -0
  27. package/dist/lib/utils/manifestWriter.js +6 -13
  28. package/package.json +8 -6
  29. package/dist/lib/config/aws/__t17fixture.js +0 -3
  30. package/dist/lib/config/aws/__t17fixtureType.d.ts +0 -2
  31. package/dist/lib/config/aws/__t17fixtureType.js +0 -1
  32. package/dist/lib/config/aws/eventBus.d.ts +0 -7
  33. package/dist/lib/config/aws/eventBus.js +0 -21
  34. package/dist/lib/config/aws/identityCenterGroupMembership.d.ts +0 -10
  35. package/dist/lib/config/aws/identityCenterGroupMembership.js +0 -102
  36. package/dist/lib/config/aws/securityBaseline.d.ts +0 -15
  37. package/dist/lib/config/aws/securityBaseline.js +0 -27
  38. package/dist/lib/patterns/aws/_eslint_test_tmp/leak.d.ts +0 -1
  39. package/dist/lib/patterns/aws/_eslint_test_tmp/leak.js +0 -4
  40. package/dist/lib/patterns/aws/managedIdentityCenter.d.ts +0 -4
  41. package/dist/lib/patterns/aws/managedIdentityCenter.js +0 -19
  42. package/dist/lib/patterns/aws/subdomainHostedZone.d.ts +0 -9
  43. package/dist/lib/patterns/aws/subdomainHostedZone.js +0 -34
  44. package/dist/lib/resources/aws/analytics/clickhouse.d.ts +0 -15
  45. package/dist/lib/resources/aws/analytics/clickhouse.js +0 -310
  46. package/dist/lib/resources/aws/analytics/clickhouseAlarms.d.ts +0 -49
  47. package/dist/lib/resources/aws/analytics/clickhouseAlarms.js +0 -140
  48. package/dist/lib/resources/aws/analytics/clickhouseConstants.d.ts +0 -73
  49. package/dist/lib/resources/aws/analytics/clickhouseConstants.js +0 -89
  50. package/dist/lib/resources/aws/analytics/clickhouseSecurityGroup.d.ts +0 -13
  51. package/dist/lib/resources/aws/analytics/clickhouseSecurityGroup.js +0 -28
  52. package/dist/lib/resources/aws/analytics/clickhouseTypes.d.ts +0 -59
  53. package/dist/lib/resources/aws/analytics/clickhouseTypes.js +0 -1
  54. package/dist/lib/resources/aws/analytics/clickhouseUserData.d.ts +0 -6
  55. package/dist/lib/resources/aws/analytics/clickhouseUserData.js +0 -299
  56. package/dist/lib/resources/aws/analytics/index.d.ts +0 -4
  57. package/dist/lib/resources/aws/analytics/index.js +0 -2
  58. package/dist/lib/resources/aws/compute/__tmp__/regression-shape.d.ts +0 -2
  59. package/dist/lib/resources/aws/compute/__tmp__/regression-shape.js +0 -11
  60. package/dist/lib/resources/aws/messaging/defaultEventBus.d.ts +0 -7
  61. package/dist/lib/resources/aws/messaging/defaultEventBus.js +0 -21
  62. package/dist/lib/resources/aws/networking/domain.d.ts +0 -13
  63. package/dist/lib/resources/aws/networking/domain.js +0 -100
  64. package/dist/lib/synth_dump.d.ts +0 -1
  65. package/dist/lib/synth_dump.js +0 -42
  66. package/dist/lib/utils/bastionFactory.d.ts +0 -10
  67. package/dist/lib/utils/bastionFactory.js +0 -29
  68. package/dist/lib/utils/constructMap.d.ts +0 -33
  69. package/dist/lib/utils/constructMap.js +0 -154
  70. package/dist/lib/utils/dnsRecords.d.ts +0 -4
  71. package/dist/lib/utils/dnsRecords.js +0 -104
  72. /package/dist/lib/{config/aws/__t17fixture.d.ts → patterns/aws/clickhouseTls/types.js} +0 -0
@@ -1,89 +0,0 @@
1
- /** Cluster/task family name used for ECS resources. */
2
- export const CLICKHOUSE_CLUSTER_NAME = "clickhouse-analytics";
3
- /** Default EC2 instance type for ClickHouse (Graviton — best cost/performance). */
4
- export const DEFAULT_CLICKHOUSE_INSTANCE_TYPE = "t4g.medium";
5
- /** ClickHouse container image. */
6
- export const CLICKHOUSE_IMAGE = "clickhouse/clickhouse-server:26.3-alpine";
7
- /** EBS volume configuration. */
8
- export const CLICKHOUSE_EBS_VOLUME_SIZE_GB = 80;
9
- export const CLICKHOUSE_EBS_IOPS = 3000;
10
- export const CLICKHOUSE_EBS_THROUGHPUT_MBPS = 125;
11
- /** ECS task resource allocation (t4g.medium = 4 GB total). */
12
- export const CLICKHOUSE_TASK_MEMORY_MIB = 3072;
13
- export const CLICKHOUSE_TASK_CPU_UNITS = 1024;
14
- /** ClickHouse ports. */
15
- export const CLICKHOUSE_HTTP_PORT = 8123;
16
- export const CLICKHOUSE_NATIVE_PORT = 9000;
17
- export const CLICKHOUSE_PROMETHEUS_PORT = 9363;
18
- /** EBS device name for the data volume (must match user data script). */
19
- export const CLICKHOUSE_EBS_DEVICE_NAME = "/dev/xvdf";
20
- /** EBS mount path on the EC2 host. */
21
- export const CLICKHOUSE_DATA_MOUNT_PATH = "/mnt/clickhouse-data";
22
- /** Secrets Manager path prefix. */
23
- export const CLICKHOUSE_SECRETS_PREFIX = "fjall/clickhouse";
24
- /** Secret names (under the prefix). */
25
- export const CLICKHOUSE_SECRET_NAMES = {
26
- APP_PASSWORD: "app-password",
27
- AUDIT_PASSWORD: "audit-password",
28
- BACKUP_PASSWORD: "backup-password",
29
- SCHEMA_PASSWORD: "schema-password"
30
- };
31
- /** Shared secret generation options (all ClickHouse users share the same policy). */
32
- export const CLICKHOUSE_SECRET_OPTIONS = {
33
- excludePunctuation: true,
34
- passwordLength: 32
35
- };
36
- /** Health check configuration. */
37
- export const CLICKHOUSE_HEALTH_CHECK = {
38
- INTERVAL_SECONDS: 30,
39
- TIMEOUT_SECONDS: 5,
40
- RETRIES: 3,
41
- START_PERIOD_SECONDS: 60
42
- };
43
- /** OPTIMIZE TABLE FINAL schedule.
44
- * RMT tables carry min_age_to_force_merge_seconds=600 so the engine already merges
45
- * old parts within 10 min; this task is a safety net for MVs (no engine-level setting)
46
- * and for ReplacingMergeTree dedup under skewed write patterns. 6 hours is sufficient. */
47
- export const OPTIMISE_FINAL_SCHEDULE = "rate(6 hours)";
48
- /** Tables requiring periodic OPTIMIZE FINAL (ReplacingMergeTree only).
49
- * Keep in sync with REPLACING_MERGE_TREE_TABLES in
50
- * webapp/app/.server/lib/clickhouse/tenantQuery.ts (auto-FINAL). */
51
- export const REPLACING_MERGE_TREE_TABLES = [
52
- "application_metrics",
53
- "cost_records",
54
- "log_fingerprints",
55
- "insights",
56
- "asset_inventory"
57
- ];
58
- /** Subdirectory on the EBS volume for server config files (must match CDK volume mount). */
59
- export const CLICKHOUSE_CONFIG_SUBDIR = "server-config.d";
60
- /** Subdirectory on the EBS volume for users config files (must match CDK volume mount). */
61
- export const CLICKHOUSE_USERS_SUBDIR = "server-users.d";
62
- /** Cloud Map namespace for ClickHouse service discovery. */
63
- export const CLICKHOUSE_CLOUDMAP_NAMESPACE = "clickhouse.local";
64
- /** Cloud Map service name (resolves to clickhouse.clickhouse.local). */
65
- export const CLICKHOUSE_CLOUDMAP_SERVICE_NAME = "clickhouse";
66
- /** Materialised views that benefit from periodic OPTIMIZE to reduce part count at read time.
67
- * These are not ReplacingMergeTree (no dedup needed) but un-merged parts force
68
- * read-time aggregation which degrades query performance. */
69
- export const OPTIMISE_MV_TABLES = [
70
- "metrics_hourly_mv",
71
- "metrics_daily_mv",
72
- "response_time_quantiles_hourly_mv",
73
- "deployment_duration_quantiles_daily_mv",
74
- "log_severity_hourly_mv",
75
- "compliance_score_daily_mv",
76
- "ai_usage_daily_mv",
77
- "finding_daily_aggregate",
78
- "insight_pattern_dismissals"
79
- ];
80
- /** Resource allocation for the lightweight optimise task. */
81
- export const OPTIMISE_TASK_MEMORY_MIB = 256;
82
- export const OPTIMISE_TASK_CPU_UNITS = 256;
83
- /** Automated backup schedule (daily 03:00 UTC — low-traffic window). */
84
- export const BACKUP_SCHEDULE = "cron(0 3 * * ? *)";
85
- /** Resource allocation for the backup task (lightweight — clickhouse-client only). */
86
- export const BACKUP_TASK_MEMORY_MIB = 256;
87
- export const BACKUP_TASK_CPU_UNITS = 256;
88
- /** Backup object expiration: 14 days (retains 14 daily snapshots). */
89
- export const BACKUP_RETENTION_DAYS = 14;
@@ -1,13 +0,0 @@
1
- import { type ISecurityGroup, type IVpc } from "aws-cdk-lib/aws-ec2";
2
- import type { Construct } from "constructs";
3
- import { SecurityGroup } from "../networking/securityGroup.js";
4
- /**
5
- * Creates the ClickHouse security group.
6
- *
7
- * Inbound:
8
- * - TCP 8123 from webapp ECS service SG (HTTP queries)
9
- *
10
- * Outbound:
11
- * - HTTPS 443 to 0.0.0.0/0 (R2 endpoint + Secrets Manager)
12
- */
13
- export declare function createClickHouseSecurityGroup(scope: Construct, vpc: IVpc, webappSecurityGroup: ISecurityGroup): SecurityGroup;
@@ -1,28 +0,0 @@
1
- import { Peer, Port } from "aws-cdk-lib/aws-ec2";
2
- import { SecurityGroup } from "../networking/securityGroup.js";
3
- import { CLICKHOUSE_HTTP_PORT, CLICKHOUSE_NATIVE_PORT } from "./clickhouseConstants.js";
4
- /**
5
- * Creates the ClickHouse security group.
6
- *
7
- * Inbound:
8
- * - TCP 8123 from webapp ECS service SG (HTTP queries)
9
- *
10
- * Outbound:
11
- * - HTTPS 443 to 0.0.0.0/0 (R2 endpoint + Secrets Manager)
12
- */
13
- export function createClickHouseSecurityGroup(scope, vpc, webappSecurityGroup) {
14
- const sg = new SecurityGroup(scope, "ClickHouseSecurityGroup", {
15
- vpc,
16
- description: "Security group for ClickHouse analytics instance",
17
- allowAllOutbound: false
18
- });
19
- // Inbound: HTTP API from webapp
20
- sg.addIngressRule(webappSecurityGroup, Port.tcp(CLICKHOUSE_HTTP_PORT), "ClickHouse HTTP API from webapp ECS service");
21
- // Inbound: Native protocol from optimise scheduled task (same SG, self-referencing)
22
- sg.addIngressRule(sg, Port.tcp(CLICKHOUSE_NATIVE_PORT), "ClickHouse native protocol from optimise task");
23
- // Outbound: HTTPS for R2 cold storage and Secrets Manager
24
- sg.addEgressRule(Peer.anyIpv4(), Port.tcp(443), "HTTPS to R2 and Secrets Manager endpoints");
25
- // Outbound: Native protocol to ClickHouse (optimise task connection)
26
- sg.addEgressRule(sg, Port.tcp(CLICKHOUSE_NATIVE_PORT), "ClickHouse native protocol for optimise task");
27
- return sg;
28
- }
@@ -1,59 +0,0 @@
1
- import type { IVpc, ISecurityGroup } from "aws-cdk-lib/aws-ec2";
2
- import type { ILogGroup } from "aws-cdk-lib/aws-logs";
3
- import type { IBucket } from "aws-cdk-lib/aws-s3";
4
- import type { ISecret } from "aws-cdk-lib/aws-secretsmanager";
5
- import type { ITopic } from "aws-cdk-lib/aws-sns";
6
- /** Props for the ClickHouse CDK construct. */
7
- export interface ClickHouseProps {
8
- /** VPC to deploy into. */
9
- vpc: IVpc;
10
- /**
11
- * EC2 instance type for ClickHouse.
12
- * Overridden by CDK context parameter `clickhouseInstanceType` if set.
13
- * Default: t4g.medium (4 GB RAM).
14
- */
15
- instanceType?: string;
16
- /**
17
- * Security group of the webapp ECS service.
18
- * Used to allow inbound HTTP (8123) from the webapp.
19
- */
20
- webappSecurityGroup: ISecurityGroup;
21
- /**
22
- * R2 configuration for cold storage.
23
- * If omitted, tiered storage is disabled (local-only).
24
- */
25
- r2Config?: ClickHouseR2Config;
26
- /**
27
- * SNS topic for CloudWatch alarms (CPU, memory, disk, stuck merges).
28
- * If omitted, posture alarms are not created.
29
- */
30
- alarmTopic?: ITopic;
31
- /**
32
- * Webapp log group, required when `alarmTopic` is set so the stuck-merge
33
- * metric filter can read the structured warning emitted by `client.ts`.
34
- */
35
- webappLogGroup?: ILogGroup;
36
- }
37
- /** Cloudflare R2 configuration for tiered storage and backups. */
38
- export interface ClickHouseR2Config {
39
- /** Cloudflare account ID. */
40
- accountId: string;
41
- /** R2 access key (stored in Secrets Manager). */
42
- accessKeySecret: ISecret;
43
- /** R2 secret key (stored in Secrets Manager). */
44
- secretKeySecret: ISecret;
45
- }
46
- /** Outputs from the ClickHouse construct for use by other constructs. */
47
- export interface ClickHouseOutputs {
48
- /** Security group for the ClickHouse instance (for connection rules). */
49
- securityGroup: ISecurityGroup;
50
- /** S3 bucket for weekly automated backups (for restore operations). */
51
- backupBucket: IBucket;
52
- /** Secrets Manager secrets for ClickHouse passwords. */
53
- secrets: {
54
- appPassword: ISecret;
55
- auditPassword: ISecret;
56
- backupPassword: ISecret;
57
- schemaPassword: ISecret;
58
- };
59
- }
@@ -1 +0,0 @@
1
- export {};
@@ -1,6 +0,0 @@
1
- export interface ClickHouseUserDataOptions {
2
- /** Cloudflare account ID for R2 cold storage. If omitted, local-only storage is used. */
3
- cfAccountId?: string;
4
- }
5
- export declare const USERS_CONFIG_XML = "<clickhouse>\n <users>\n <default>\n <networks>\n <ip>127.0.0.1</ip>\n <ip>::1</ip>\n </networks>\n </default>\n </users>\n <profiles>\n <default>\n <optimize_move_to_prewhere>1</optimize_move_to_prewhere>\n <!-- ALTER TABLE ... MODIFY TTL on a 30-day-partitioned table would otherwise\n trigger an immediate full-table rewrite (default = 1). On the t4g.medium\n box that's a merge-pool starvation event. Keep TTL changes lazy: parts\n re-evaluate TTL on their next natural merge, no forced rewrite. -->\n <materialize_ttl_after_modify>0</materialize_ttl_after_modify>\n </default>\n <app_writer>\n <max_threads>2</max_threads>\n <max_insert_threads>1</max_insert_threads>\n <max_concurrent_queries_for_user>4</max_concurrent_queries_for_user>\n <log_queries_min_query_duration_ms>100</log_queries_min_query_duration_ms>\n <optimize_move_to_prewhere>1</optimize_move_to_prewhere>\n <use_query_condition_cache>1</use_query_condition_cache>\n <!-- Re-enable skip indexes under FINAL (tenantQuery auto-FINALs RMT tables;\n default disables idx_aws_account, idx_application, idx_dedup, idx_fingerprint). -->\n <use_skip_indexes_if_final>1</use_skip_indexes_if_final>\n <!-- Tenant-isolation guards (ClickHouse PR #91065 fix). Belt-and-braces with the\n per-user SQL SETTINGS in 002-users.sql \u2014 keep both so `CREATE OR REPLACE USER`\n cannot regress this. Without these flags, FINAL queries on un-merged\n ReplacingMergeTree parts can leak across tenants. -->\n <apply_row_policy_after_final>1</apply_row_policy_after_final>\n <apply_prewhere_after_final>1</apply_prewhere_after_final>\n <do_not_merge_across_partitions_select_final>1</do_not_merge_across_partitions_select_final>\n <async_insert>1</async_insert>\n <wait_for_async_insert>1</wait_for_async_insert>\n <async_insert_max_data_size>10000000</async_insert_max_data_size>\n <!-- Adaptive batching: tune flush window between 50 ms (low-latency rare inserts)\n and 2 s (absorbs bursts). A single fixed value is silently overridden by the\n adaptive algorithm. -->\n <async_insert_busy_timeout_min_ms>50</async_insert_busy_timeout_min_ms>\n <async_insert_busy_timeout_max_ms>2000</async_insert_busy_timeout_max_ms>\n <async_insert_use_adaptive_busy_timeout>1</async_insert_use_adaptive_busy_timeout>\n <!-- Server-side deduplication of async inserts. Latent retry safety net:\n if a producer retries the same insert window (network hiccup, lambda re-run,\n SQS redelivery), the second attempt collapses against the first. As of CH 26.1\n this also propagates end-to-end through dependent materialised views \u2014 without\n it, a retried insert could double-count in metrics_hourly_mv / log_severity_hourly_mv\n even if the base table dedups. CH pin is 26.3 so the propagation fix is in. -->\n <async_insert_deduplicate>1</async_insert_deduplicate>\n <input_format_parallel_parsing>0</input_format_parallel_parsing>\n <output_format_parallel_formatting>0</output_format_parallel_formatting>\n <!-- Lazy materialisation (CH 25.4+): for `SELECT * ... LIMIT N` shapes the planner\n reads only the columns needed to evaluate ORDER BY / WHERE, then fetches the\n remaining columns for the surviving N rows. Order-of-magnitude I/O reduction\n on dashboard queries (e.g. getLatestMetrics LIMIT 1 BY application_id). -->\n <query_plan_optimize_lazy_materialization>1</query_plan_optimize_lazy_materialization>\n <!-- Per-query memory cap (overrides server-wide max_memory_usage of 1 GB\n to give app_writer 2 GB headroom). Belt-and-braces with the inline\n SETTINGS in 002-users.sql so neither layer can drift alone. -->\n <max_memory_usage>2000000000</max_memory_usage>\n <max_memory_usage_for_user>2684354560</max_memory_usage_for_user>\n <max_bytes_before_external_sort>536870912</max_bytes_before_external_sort>\n <max_bytes_before_external_group_by>536870912</max_bytes_before_external_group_by>\n <!-- Per-query caps. Belt-and-braces with the inline SETTINGS in\n 002-users.sql so `CREATE OR REPLACE USER` cannot regress the bound. -->\n <max_execution_time>30</max_execution_time>\n <max_rows_to_read>10000000</max_rows_to_read>\n </app_writer>\n <audit_writer>\n <max_threads>1</max_threads>\n <max_insert_threads>1</max_insert_threads>\n <max_concurrent_queries_for_user>2</max_concurrent_queries_for_user>\n <max_memory_usage>500000000</max_memory_usage>\n <max_execution_time>10</max_execution_time>\n <async_insert>1</async_insert>\n <wait_for_async_insert>1</wait_for_async_insert>\n </audit_writer>\n <backup_reader>\n <max_threads>2</max_threads>\n <max_concurrent_queries_for_user>1</max_concurrent_queries_for_user>\n <max_memory_usage>1000000000</max_memory_usage>\n <max_execution_time>3600</max_execution_time>\n </backup_reader>\n <schema_admin>\n <max_threads>2</max_threads>\n <max_concurrent_queries_for_user>1</max_concurrent_queries_for_user>\n <max_memory_usage>1000000000</max_memory_usage>\n <max_execution_time>1800</max_execution_time>\n </schema_admin>\n </profiles>\n <quotas>\n <tenant_default>\n <interval>\n <duration>3600</duration>\n <queries>1000</queries>\n <result_rows>10000000</result_rows>\n </interval>\n </tenant_default>\n </quotas>\n</clickhouse>";
6
- export declare function generateClickHouseUserData(options?: ClickHouseUserDataOptions): string;
@@ -1,299 +0,0 @@
1
- import { CLICKHOUSE_DATA_MOUNT_PATH, CLICKHOUSE_EBS_DEVICE_NAME, CLICKHOUSE_CONFIG_SUBDIR, CLICKHOUSE_USERS_SUBDIR, CLICKHOUSE_HTTP_PORT, CLICKHOUSE_PROMETHEUS_PORT } from "./clickhouseConstants.js";
2
- function generateServerConfigXml(cfAccountId) {
3
- if (cfAccountId !== undefined && !/^[a-f0-9]{32}$/.test(cfAccountId)) {
4
- throw new Error(`Invalid Cloudflare account ID format: expected 32-char hex, got "${cfAccountId}"`);
5
- }
6
- const storageBlock = cfAccountId !== undefined
7
- ? ` <storage_configuration>
8
- <disks>
9
- <local_ssd>
10
- <type>local</type>
11
- <path>/var/lib/clickhouse/</path>
12
- </local_ssd>
13
- <r2_cold>
14
- <type>s3</type>
15
- <endpoint>https://${cfAccountId}.r2.cloudflarestorage.com/fjall-clickhouse-cold/data/</endpoint>
16
- <access_key_id from_env="R2_ACCESS_KEY" />
17
- <secret_access_key from_env="R2_SECRET_KEY" />
18
- <region>auto</region>
19
- <support_batch_delete>false</support_batch_delete>
20
- <metadata_path>/var/lib/clickhouse/disks/r2_cold/</metadata_path>
21
- </r2_cold>
22
- </disks>
23
- <policies>
24
- <tiered>
25
- <volumes>
26
- <hot>
27
- <disk>local_ssd</disk>
28
- </hot>
29
- <cold>
30
- <disk>r2_cold</disk>
31
- </cold>
32
- </volumes>
33
- <move_factor>0.05</move_factor>
34
- </tiered>
35
- <local_only>
36
- <volumes>
37
- <default>
38
- <disk>local_ssd</disk>
39
- </default>
40
- </volumes>
41
- </local_only>
42
- </policies>
43
- </storage_configuration>`
44
- : ` <storage_configuration>
45
- <disks>
46
- <default>
47
- <keep_free_space_bytes>1073741824</keep_free_space_bytes>
48
- </default>
49
- </disks>
50
- <policies>
51
- <tiered>
52
- <volumes>
53
- <hot>
54
- <disk>default</disk>
55
- </hot>
56
- <cold>
57
- <disk>default</disk>
58
- </cold>
59
- </volumes>
60
- </tiered>
61
- <local_only>
62
- <volumes>
63
- <default>
64
- <disk>default</disk>
65
- </default>
66
- </volumes>
67
- </local_only>
68
- </policies>
69
- </storage_configuration>`;
70
- return `<clickhouse>
71
- <logger>
72
- <level>warning</level>
73
- <log>/var/log/clickhouse-server/clickhouse-server.log</log>
74
- <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
75
- <size>100M</size>
76
- <count>3</count>
77
- </logger>
78
- <max_server_memory_usage>2684354560</max_server_memory_usage>
79
- <max_memory_usage>1000000000</max_memory_usage>
80
- <max_bytes_before_external_sort>536870912</max_bytes_before_external_sort>
81
- <max_bytes_before_external_group_by>536870912</max_bytes_before_external_group_by>
82
- <mark_cache_size>268435456</mark_cache_size>
83
- <index_mark_cache_size>67108864</index_mark_cache_size>
84
- <uncompressed_cache_size>0</uncompressed_cache_size>
85
- <max_concurrent_queries>10</max_concurrent_queries>
86
- <background_pool_size>2</background_pool_size>
87
- <background_schedule_pool_size>2</background_schedule_pool_size>
88
- <background_merges_mutations_concurrency_ratio>2</background_merges_mutations_concurrency_ratio>
89
- <background_move_pool_size>1</background_move_pool_size>
90
- <background_fetches_pool_size>1</background_fetches_pool_size>
91
- <background_message_broker_schedule_pool_size>1</background_message_broker_schedule_pool_size>
92
- <merge_tree>
93
- <max_suspicious_broken_parts>5</max_suspicious_broken_parts>
94
- <parts_to_delay_insert>150</parts_to_delay_insert>
95
- <parts_to_throw_insert>300</parts_to_throw_insert>
96
- </merge_tree>
97
- <http_port>${CLICKHOUSE_HTTP_PORT}</http_port>
98
- <custom_settings_prefixes>current_</custom_settings_prefixes>
99
- <!-- HTTP keep-alive window. Must exceed @clickhouse/client idle_socket_ttl (15 s)
100
- so the client always closes the socket first. Prevents ECONNRESET on reuse. -->
101
- <keep_alive_timeout>30</keep_alive_timeout>
102
- <query_log>
103
- <database>system</database>
104
- <table>query_log</table>
105
- <flush_interval_milliseconds>7500</flush_interval_milliseconds>
106
- <ttl>event_date + INTERVAL 14 DAY DELETE</ttl>
107
- </query_log>
108
- <!-- All system log tables need bounded retention on 80GB EBS (mirror query_log's 14-day policy). -->
109
- <asynchronous_metric_log>
110
- <database>system</database>
111
- <table>asynchronous_metric_log</table>
112
- <ttl>event_date + INTERVAL 14 DAY DELETE</ttl>
113
- </asynchronous_metric_log>
114
- <metric_log>
115
- <database>system</database>
116
- <table>metric_log</table>
117
- <ttl>event_date + INTERVAL 14 DAY DELETE</ttl>
118
- </metric_log>
119
- <part_log>
120
- <database>system</database>
121
- <table>part_log</table>
122
- <ttl>event_date + INTERVAL 14 DAY DELETE</ttl>
123
- </part_log>
124
- <error_log>
125
- <database>system</database>
126
- <table>error_log</table>
127
- <ttl>event_date + INTERVAL 14 DAY DELETE</ttl>
128
- </error_log>
129
- ${storageBlock}
130
- <query_cache>
131
- <max_size_in_bytes>104857600</max_size_in_bytes>
132
- <max_entries>1024</max_entries>
133
- <max_entry_size_in_bytes>1048576</max_entry_size_in_bytes>
134
- </query_cache>
135
- <prometheus>
136
- <endpoint>/metrics</endpoint>
137
- <port>${CLICKHOUSE_PROMETHEUS_PORT}</port>
138
- <metrics>true</metrics>
139
- <events>true</events>
140
- <asynchronous_metrics>true</asynchronous_metrics>
141
- </prometheus>
142
- <query_thread_log remove="1"/>
143
- <opentelemetry_span_log remove="1"/>
144
- <processors_profile_log remove="1"/>
145
- </clickhouse>`;
146
- }
147
- export const USERS_CONFIG_XML = `<clickhouse>
148
- <users>
149
- <default>
150
- <networks>
151
- <ip>127.0.0.1</ip>
152
- <ip>::1</ip>
153
- </networks>
154
- </default>
155
- </users>
156
- <profiles>
157
- <default>
158
- <optimize_move_to_prewhere>1</optimize_move_to_prewhere>
159
- <!-- ALTER TABLE ... MODIFY TTL on a 30-day-partitioned table would otherwise
160
- trigger an immediate full-table rewrite (default = 1). On the t4g.medium
161
- box that's a merge-pool starvation event. Keep TTL changes lazy: parts
162
- re-evaluate TTL on their next natural merge, no forced rewrite. -->
163
- <materialize_ttl_after_modify>0</materialize_ttl_after_modify>
164
- </default>
165
- <app_writer>
166
- <max_threads>2</max_threads>
167
- <max_insert_threads>1</max_insert_threads>
168
- <max_concurrent_queries_for_user>4</max_concurrent_queries_for_user>
169
- <log_queries_min_query_duration_ms>100</log_queries_min_query_duration_ms>
170
- <optimize_move_to_prewhere>1</optimize_move_to_prewhere>
171
- <use_query_condition_cache>1</use_query_condition_cache>
172
- <!-- Re-enable skip indexes under FINAL (tenantQuery auto-FINALs RMT tables;
173
- default disables idx_aws_account, idx_application, idx_dedup, idx_fingerprint). -->
174
- <use_skip_indexes_if_final>1</use_skip_indexes_if_final>
175
- <!-- Tenant-isolation guards (ClickHouse PR #91065 fix). Belt-and-braces with the
176
- per-user SQL SETTINGS in 002-users.sql — keep both so \`CREATE OR REPLACE USER\`
177
- cannot regress this. Without these flags, FINAL queries on un-merged
178
- ReplacingMergeTree parts can leak across tenants. -->
179
- <apply_row_policy_after_final>1</apply_row_policy_after_final>
180
- <apply_prewhere_after_final>1</apply_prewhere_after_final>
181
- <do_not_merge_across_partitions_select_final>1</do_not_merge_across_partitions_select_final>
182
- <async_insert>1</async_insert>
183
- <wait_for_async_insert>1</wait_for_async_insert>
184
- <async_insert_max_data_size>10000000</async_insert_max_data_size>
185
- <!-- Adaptive batching: tune flush window between 50 ms (low-latency rare inserts)
186
- and 2 s (absorbs bursts). A single fixed value is silently overridden by the
187
- adaptive algorithm. -->
188
- <async_insert_busy_timeout_min_ms>50</async_insert_busy_timeout_min_ms>
189
- <async_insert_busy_timeout_max_ms>2000</async_insert_busy_timeout_max_ms>
190
- <async_insert_use_adaptive_busy_timeout>1</async_insert_use_adaptive_busy_timeout>
191
- <!-- Server-side deduplication of async inserts. Latent retry safety net:
192
- if a producer retries the same insert window (network hiccup, lambda re-run,
193
- SQS redelivery), the second attempt collapses against the first. As of CH 26.1
194
- this also propagates end-to-end through dependent materialised views — without
195
- it, a retried insert could double-count in metrics_hourly_mv / log_severity_hourly_mv
196
- even if the base table dedups. CH pin is 26.3 so the propagation fix is in. -->
197
- <async_insert_deduplicate>1</async_insert_deduplicate>
198
- <input_format_parallel_parsing>0</input_format_parallel_parsing>
199
- <output_format_parallel_formatting>0</output_format_parallel_formatting>
200
- <!-- Lazy materialisation (CH 25.4+): for \`SELECT * ... LIMIT N\` shapes the planner
201
- reads only the columns needed to evaluate ORDER BY / WHERE, then fetches the
202
- remaining columns for the surviving N rows. Order-of-magnitude I/O reduction
203
- on dashboard queries (e.g. getLatestMetrics LIMIT 1 BY application_id). -->
204
- <query_plan_optimize_lazy_materialization>1</query_plan_optimize_lazy_materialization>
205
- <!-- Per-query memory cap (overrides server-wide max_memory_usage of 1 GB
206
- to give app_writer 2 GB headroom). Belt-and-braces with the inline
207
- SETTINGS in 002-users.sql so neither layer can drift alone. -->
208
- <max_memory_usage>2000000000</max_memory_usage>
209
- <max_memory_usage_for_user>2684354560</max_memory_usage_for_user>
210
- <max_bytes_before_external_sort>536870912</max_bytes_before_external_sort>
211
- <max_bytes_before_external_group_by>536870912</max_bytes_before_external_group_by>
212
- <!-- Per-query caps. Belt-and-braces with the inline SETTINGS in
213
- 002-users.sql so \`CREATE OR REPLACE USER\` cannot regress the bound. -->
214
- <max_execution_time>30</max_execution_time>
215
- <max_rows_to_read>10000000</max_rows_to_read>
216
- </app_writer>
217
- <audit_writer>
218
- <max_threads>1</max_threads>
219
- <max_insert_threads>1</max_insert_threads>
220
- <max_concurrent_queries_for_user>2</max_concurrent_queries_for_user>
221
- <max_memory_usage>500000000</max_memory_usage>
222
- <max_execution_time>10</max_execution_time>
223
- <async_insert>1</async_insert>
224
- <wait_for_async_insert>1</wait_for_async_insert>
225
- </audit_writer>
226
- <backup_reader>
227
- <max_threads>2</max_threads>
228
- <max_concurrent_queries_for_user>1</max_concurrent_queries_for_user>
229
- <max_memory_usage>1000000000</max_memory_usage>
230
- <max_execution_time>3600</max_execution_time>
231
- </backup_reader>
232
- <schema_admin>
233
- <max_threads>2</max_threads>
234
- <max_concurrent_queries_for_user>1</max_concurrent_queries_for_user>
235
- <max_memory_usage>1000000000</max_memory_usage>
236
- <max_execution_time>1800</max_execution_time>
237
- </schema_admin>
238
- </profiles>
239
- <quotas>
240
- <tenant_default>
241
- <interval>
242
- <duration>3600</duration>
243
- <queries>1000</queries>
244
- <result_rows>10000000</result_rows>
245
- </interval>
246
- </tenant_default>
247
- </quotas>
248
- </clickhouse>`;
249
- export function generateClickHouseUserData(options) {
250
- const serverConfigXml = generateServerConfigXml(options?.cfAccountId);
251
- return `#!/bin/bash
252
- set -euo pipefail
253
-
254
- DEVICE="${CLICKHOUSE_EBS_DEVICE_NAME}"
255
- MOUNT_POINT="${CLICKHOUSE_DATA_MOUNT_PATH}"
256
-
257
- # Wait for the EBS volume to be attached (up to 60 seconds)
258
- for i in $(seq 1 60); do
259
- if [ -b "$DEVICE" ]; then
260
- break
261
- fi
262
- sleep 1
263
- done
264
-
265
- if [ ! -b "$DEVICE" ]; then
266
- echo "ERROR: EBS volume $DEVICE not found after 60 seconds" >&2
267
- exit 1
268
- fi
269
-
270
- # Format only if not already formatted (idempotent)
271
- if ! blkid "$DEVICE" | grep -q ext4; then
272
- mkfs.ext4 -m 0 "$DEVICE"
273
- fi
274
-
275
- # Mount
276
- mkdir -p "$MOUNT_POINT"
277
- mount "$DEVICE" "$MOUNT_POINT"
278
-
279
- # Persist across reboots
280
- if ! grep -q "$MOUNT_POINT" /etc/fstab; then
281
- echo "$DEVICE $MOUNT_POINT ext4 defaults,nofail 0 2" >> /etc/fstab
282
- fi
283
-
284
- # Write ClickHouse server config (memory limits, storage policies, Prometheus)
285
- mkdir -p "$MOUNT_POINT/${CLICKHOUSE_CONFIG_SUBDIR}"
286
- cat > "$MOUNT_POINT/${CLICKHOUSE_CONFIG_SUBDIR}/fjall.xml" << 'CONFIGEOF'
287
- ${serverConfigXml}
288
- CONFIGEOF
289
-
290
- # Write ClickHouse users config (profiles, quotas, default user restrictions)
291
- mkdir -p "$MOUNT_POINT/${CLICKHOUSE_USERS_SUBDIR}"
292
- cat > "$MOUNT_POINT/${CLICKHOUSE_USERS_SUBDIR}/fjall.xml" << 'USERSEOF'
293
- ${USERS_CONFIG_XML}
294
- USERSEOF
295
-
296
- # ClickHouse alpine image runs as uid 101, gid 101
297
- chown -R 101:101 "$MOUNT_POINT"
298
- `;
299
- }
@@ -1,4 +0,0 @@
1
- export { default as ClickHouse } from "./clickhouse.js";
2
- export type { ClickHouseProps, ClickHouseR2Config, ClickHouseOutputs } from "./clickhouseTypes.js";
3
- export { createClickHouseAlarms } from "./clickhouseAlarms.js";
4
- export type { ClickHouseAlarmThresholds, ClickHouseAlarmsProps } from "./clickhouseAlarms.js";
@@ -1,2 +0,0 @@
1
- export { default as ClickHouse } from "./clickhouse.js";
2
- export { createClickHouseAlarms } from "./clickhouseAlarms.js";
@@ -1,2 +0,0 @@
1
- import { AutoScalingGroup } from "aws-cdk-lib/aws-autoscaling";
2
- export declare function regression(asg: AutoScalingGroup, queue: any, id: string): void;
@@ -1,11 +0,0 @@
1
- import { DefaultResult, LifecycleTransition } from "aws-cdk-lib/aws-autoscaling";
2
- import { QueueHook } from "aws-cdk-lib/aws-autoscaling-hooktargets";
3
- import { Duration } from "aws-cdk-lib";
4
- export function regression(asg, queue, id) {
5
- asg.addLifecycleHook(`${id}LaunchingHook`, {
6
- lifecycleTransition: LifecycleTransition.INSTANCE_LAUNCHING,
7
- defaultResult: DefaultResult.ABANDON,
8
- heartbeatTimeout: Duration.seconds(300),
9
- notificationTarget: new QueueHook(queue)
10
- });
11
- }
@@ -1,7 +0,0 @@
1
- import { CfnOutput } from "aws-cdk-lib";
2
- import { Construct } from "constructs";
3
- export declare class DefaultEventBus extends Construct {
4
- readonly defaultEventBusName: CfnOutput;
5
- readonly defaultEventBusArn: CfnOutput;
6
- constructor(scope: Construct, id: string);
7
- }
@@ -1,21 +0,0 @@
1
- import { CfnOutput } from "aws-cdk-lib";
2
- import { EventBus } from "aws-cdk-lib/aws-events";
3
- import { Construct } from "constructs";
4
- export class DefaultEventBus extends Construct {
5
- defaultEventBusName;
6
- defaultEventBusArn;
7
- constructor(scope, id) {
8
- super(scope, id);
9
- const eventBridge = EventBus.fromEventBusName(this, "DefaultEventBus", "default");
10
- this.defaultEventBusName = new CfnOutput(this, "DefaultEventBusNameOutput", {
11
- key: "DefaultEventBusName",
12
- value: eventBridge.eventBusName,
13
- exportName: "DefaultEventBusName"
14
- });
15
- this.defaultEventBusArn = new CfnOutput(this, "DefaultEventBusArnOutput", {
16
- key: "DefaultEventBusArn",
17
- value: eventBridge.eventBusArn,
18
- exportName: "DefaultEventBusArn"
19
- });
20
- }
21
- }
@@ -1,13 +0,0 @@
1
- import { Construct } from "constructs";
2
- import { type IHostedZone } from "aws-cdk-lib/aws-route53";
3
- import type { DomainApexProps } from "../../../utils/domainTypes.js";
4
- export declare class Domain extends Construct {
5
- readonly hostedZoneId: string;
6
- readonly hostedZone: IHostedZone;
7
- private readonly zoneName;
8
- constructor(scope: Construct, id: string, props: DomainApexProps);
9
- private addDelegationRole;
10
- private addZoneIdOutput;
11
- private addNameserverOutput;
12
- private addCertificates;
13
- }