@fjall/components-infrastructure 0.102.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/dist/lib/lambda-assets/cert-generator/asset/index.js +17948 -0
  2. package/dist/lib/lambda-assets/cert-generator/asset/package.json +4 -0
  3. package/dist/lib/patterns/aws/clickhouseDatabase.d.ts +37 -0
  4. package/dist/lib/patterns/aws/clickhouseDatabase.js +120 -19
  5. package/dist/lib/patterns/aws/clickhouseTls/index.d.ts +1 -0
  6. package/dist/lib/patterns/aws/clickhouseTls/index.js +1 -0
  7. package/dist/lib/patterns/aws/clickhouseTls/types.d.ts +48 -0
  8. package/dist/lib/resources/aws/database/clickhouseConstants.d.ts +21 -0
  9. package/dist/lib/resources/aws/database/clickhouseConstants.js +21 -0
  10. package/dist/lib/resources/aws/database/clickhouseSecurityGroup.d.ts +2 -0
  11. package/dist/lib/resources/aws/database/clickhouseSecurityGroup.js +2 -0
  12. package/dist/lib/resources/aws/database/clickhouseUserData.d.ts +21 -0
  13. package/dist/lib/resources/aws/database/clickhouseUserData.js +48 -3
  14. package/dist/lib/resources/aws/database/clickhouseXmlRenderer.d.ts +1 -1
  15. package/dist/lib/resources/aws/database/clickhouseXmlRenderer.js +1 -1
  16. package/dist/lib/resources/aws/secrets/index.d.ts +2 -0
  17. package/dist/lib/resources/aws/secrets/index.js +2 -0
  18. package/dist/lib/resources/aws/secrets/tlsCaSecret.d.ts +13 -0
  19. package/dist/lib/resources/aws/secrets/tlsCaSecret.js +15 -0
  20. package/dist/lib/resources/aws/secrets/tlsServerSecret.d.ts +15 -0
  21. package/dist/lib/resources/aws/secrets/tlsServerSecret.js +17 -0
  22. package/dist/lib/resources/aws/utilities/index.d.ts +1 -0
  23. package/dist/lib/resources/aws/utilities/index.js +1 -0
  24. package/dist/lib/resources/aws/utilities/tlsCertGenerator.d.ts +33 -0
  25. package/dist/lib/resources/aws/utilities/tlsCertGenerator.js +67 -0
  26. package/package.json +7 -5
  27. package/dist/lib/config/aws/__t17fixture.js +0 -3
  28. package/dist/lib/config/aws/__t17fixtureType.d.ts +0 -2
  29. package/dist/lib/config/aws/__t17fixtureType.js +0 -1
  30. package/dist/lib/config/aws/eventBus.d.ts +0 -7
  31. package/dist/lib/config/aws/eventBus.js +0 -21
  32. package/dist/lib/config/aws/identityCenterGroupMembership.d.ts +0 -10
  33. package/dist/lib/config/aws/identityCenterGroupMembership.js +0 -102
  34. package/dist/lib/config/aws/securityBaseline.d.ts +0 -15
  35. package/dist/lib/config/aws/securityBaseline.js +0 -27
  36. package/dist/lib/patterns/aws/_eslint_test_tmp/leak.d.ts +0 -1
  37. package/dist/lib/patterns/aws/_eslint_test_tmp/leak.js +0 -4
  38. package/dist/lib/patterns/aws/managedIdentityCenter.d.ts +0 -4
  39. package/dist/lib/patterns/aws/managedIdentityCenter.js +0 -19
  40. package/dist/lib/patterns/aws/subdomainHostedZone.d.ts +0 -9
  41. package/dist/lib/patterns/aws/subdomainHostedZone.js +0 -34
  42. package/dist/lib/resources/aws/analytics/clickhouse.d.ts +0 -15
  43. package/dist/lib/resources/aws/analytics/clickhouse.js +0 -310
  44. package/dist/lib/resources/aws/analytics/clickhouseAlarms.d.ts +0 -49
  45. package/dist/lib/resources/aws/analytics/clickhouseAlarms.js +0 -140
  46. package/dist/lib/resources/aws/analytics/clickhouseConstants.d.ts +0 -73
  47. package/dist/lib/resources/aws/analytics/clickhouseConstants.js +0 -89
  48. package/dist/lib/resources/aws/analytics/clickhouseSecurityGroup.d.ts +0 -13
  49. package/dist/lib/resources/aws/analytics/clickhouseSecurityGroup.js +0 -28
  50. package/dist/lib/resources/aws/analytics/clickhouseTypes.d.ts +0 -59
  51. package/dist/lib/resources/aws/analytics/clickhouseTypes.js +0 -1
  52. package/dist/lib/resources/aws/analytics/clickhouseUserData.d.ts +0 -6
  53. package/dist/lib/resources/aws/analytics/clickhouseUserData.js +0 -299
  54. package/dist/lib/resources/aws/analytics/index.d.ts +0 -4
  55. package/dist/lib/resources/aws/analytics/index.js +0 -2
  56. package/dist/lib/resources/aws/compute/__tmp__/regression-shape.d.ts +0 -2
  57. package/dist/lib/resources/aws/compute/__tmp__/regression-shape.js +0 -11
  58. package/dist/lib/resources/aws/messaging/defaultEventBus.d.ts +0 -7
  59. package/dist/lib/resources/aws/messaging/defaultEventBus.js +0 -21
  60. package/dist/lib/resources/aws/networking/domain.d.ts +0 -13
  61. package/dist/lib/resources/aws/networking/domain.js +0 -100
  62. package/dist/lib/synth_dump.d.ts +0 -1
  63. package/dist/lib/synth_dump.js +0 -42
  64. package/dist/lib/utils/bastionFactory.d.ts +0 -10
  65. package/dist/lib/utils/bastionFactory.js +0 -29
  66. package/dist/lib/utils/constructMap.d.ts +0 -33
  67. package/dist/lib/utils/constructMap.js +0 -154
  68. package/dist/lib/utils/dnsRecords.d.ts +0 -4
  69. package/dist/lib/utils/dnsRecords.js +0 -104
  70. /package/dist/lib/{config/aws/__t17fixture.d.ts → patterns/aws/clickhouseTls/types.js} +0 -0
@@ -1,13 +0,0 @@
1
- import { type ISecurityGroup, type IVpc } from "aws-cdk-lib/aws-ec2";
2
- import type { Construct } from "constructs";
3
- import { SecurityGroup } from "../networking/securityGroup.js";
4
- /**
5
- * Creates the ClickHouse security group.
6
- *
7
- * Inbound:
8
- * - TCP 8123 from webapp ECS service SG (HTTP queries)
9
- *
10
- * Outbound:
11
- * - HTTPS 443 to 0.0.0.0/0 (R2 endpoint + Secrets Manager)
12
- */
13
- export declare function createClickHouseSecurityGroup(scope: Construct, vpc: IVpc, webappSecurityGroup: ISecurityGroup): SecurityGroup;
@@ -1,28 +0,0 @@
1
- import { Peer, Port } from "aws-cdk-lib/aws-ec2";
2
- import { SecurityGroup } from "../networking/securityGroup.js";
3
- import { CLICKHOUSE_HTTP_PORT, CLICKHOUSE_NATIVE_PORT } from "./clickhouseConstants.js";
4
- /**
5
- * Creates the ClickHouse security group.
6
- *
7
- * Inbound:
8
- * - TCP 8123 from webapp ECS service SG (HTTP queries)
9
- *
10
- * Outbound:
11
- * - HTTPS 443 to 0.0.0.0/0 (R2 endpoint + Secrets Manager)
12
- */
13
- export function createClickHouseSecurityGroup(scope, vpc, webappSecurityGroup) {
14
- const sg = new SecurityGroup(scope, "ClickHouseSecurityGroup", {
15
- vpc,
16
- description: "Security group for ClickHouse analytics instance",
17
- allowAllOutbound: false
18
- });
19
- // Inbound: HTTP API from webapp
20
- sg.addIngressRule(webappSecurityGroup, Port.tcp(CLICKHOUSE_HTTP_PORT), "ClickHouse HTTP API from webapp ECS service");
21
- // Inbound: Native protocol from optimise scheduled task (same SG, self-referencing)
22
- sg.addIngressRule(sg, Port.tcp(CLICKHOUSE_NATIVE_PORT), "ClickHouse native protocol from optimise task");
23
- // Outbound: HTTPS for R2 cold storage and Secrets Manager
24
- sg.addEgressRule(Peer.anyIpv4(), Port.tcp(443), "HTTPS to R2 and Secrets Manager endpoints");
25
- // Outbound: Native protocol to ClickHouse (optimise task connection)
26
- sg.addEgressRule(sg, Port.tcp(CLICKHOUSE_NATIVE_PORT), "ClickHouse native protocol for optimise task");
27
- return sg;
28
- }
@@ -1,59 +0,0 @@
1
- import type { IVpc, ISecurityGroup } from "aws-cdk-lib/aws-ec2";
2
- import type { ILogGroup } from "aws-cdk-lib/aws-logs";
3
- import type { IBucket } from "aws-cdk-lib/aws-s3";
4
- import type { ISecret } from "aws-cdk-lib/aws-secretsmanager";
5
- import type { ITopic } from "aws-cdk-lib/aws-sns";
6
- /** Props for the ClickHouse CDK construct. */
7
- export interface ClickHouseProps {
8
- /** VPC to deploy into. */
9
- vpc: IVpc;
10
- /**
11
- * EC2 instance type for ClickHouse.
12
- * Overridden by CDK context parameter `clickhouseInstanceType` if set.
13
- * Default: t4g.medium (4 GB RAM).
14
- */
15
- instanceType?: string;
16
- /**
17
- * Security group of the webapp ECS service.
18
- * Used to allow inbound HTTP (8123) from the webapp.
19
- */
20
- webappSecurityGroup: ISecurityGroup;
21
- /**
22
- * R2 configuration for cold storage.
23
- * If omitted, tiered storage is disabled (local-only).
24
- */
25
- r2Config?: ClickHouseR2Config;
26
- /**
27
- * SNS topic for CloudWatch alarms (CPU, memory, disk, stuck merges).
28
- * If omitted, posture alarms are not created.
29
- */
30
- alarmTopic?: ITopic;
31
- /**
32
- * Webapp log group, required when `alarmTopic` is set so the stuck-merge
33
- * metric filter can read the structured warning emitted by `client.ts`.
34
- */
35
- webappLogGroup?: ILogGroup;
36
- }
37
- /** Cloudflare R2 configuration for tiered storage and backups. */
38
- export interface ClickHouseR2Config {
39
- /** Cloudflare account ID. */
40
- accountId: string;
41
- /** R2 access key (stored in Secrets Manager). */
42
- accessKeySecret: ISecret;
43
- /** R2 secret key (stored in Secrets Manager). */
44
- secretKeySecret: ISecret;
45
- }
46
- /** Outputs from the ClickHouse construct for use by other constructs. */
47
- export interface ClickHouseOutputs {
48
- /** Security group for the ClickHouse instance (for connection rules). */
49
- securityGroup: ISecurityGroup;
50
- /** S3 bucket for weekly automated backups (for restore operations). */
51
- backupBucket: IBucket;
52
- /** Secrets Manager secrets for ClickHouse passwords. */
53
- secrets: {
54
- appPassword: ISecret;
55
- auditPassword: ISecret;
56
- backupPassword: ISecret;
57
- schemaPassword: ISecret;
58
- };
59
- }
@@ -1 +0,0 @@
1
- export {};
@@ -1,6 +0,0 @@
1
- export interface ClickHouseUserDataOptions {
2
- /** Cloudflare account ID for R2 cold storage. If omitted, local-only storage is used. */
3
- cfAccountId?: string;
4
- }
5
- export declare const USERS_CONFIG_XML = "<clickhouse>\n <users>\n <default>\n <networks>\n <ip>127.0.0.1</ip>\n <ip>::1</ip>\n </networks>\n </default>\n </users>\n <profiles>\n <default>\n <optimize_move_to_prewhere>1</optimize_move_to_prewhere>\n <!-- ALTER TABLE ... MODIFY TTL on a 30-day-partitioned table would otherwise\n trigger an immediate full-table rewrite (default = 1). On the t4g.medium\n box that's a merge-pool starvation event. Keep TTL changes lazy: parts\n re-evaluate TTL on their next natural merge, no forced rewrite. -->\n <materialize_ttl_after_modify>0</materialize_ttl_after_modify>\n </default>\n <app_writer>\n <max_threads>2</max_threads>\n <max_insert_threads>1</max_insert_threads>\n <max_concurrent_queries_for_user>4</max_concurrent_queries_for_user>\n <log_queries_min_query_duration_ms>100</log_queries_min_query_duration_ms>\n <optimize_move_to_prewhere>1</optimize_move_to_prewhere>\n <use_query_condition_cache>1</use_query_condition_cache>\n <!-- Re-enable skip indexes under FINAL (tenantQuery auto-FINALs RMT tables;\n default disables idx_aws_account, idx_application, idx_dedup, idx_fingerprint). -->\n <use_skip_indexes_if_final>1</use_skip_indexes_if_final>\n <!-- Tenant-isolation guards (ClickHouse PR #91065 fix). Belt-and-braces with the\n per-user SQL SETTINGS in 002-users.sql \u2014 keep both so `CREATE OR REPLACE USER`\n cannot regress this. Without these flags, FINAL queries on un-merged\n ReplacingMergeTree parts can leak across tenants. -->\n <apply_row_policy_after_final>1</apply_row_policy_after_final>\n <apply_prewhere_after_final>1</apply_prewhere_after_final>\n <do_not_merge_across_partitions_select_final>1</do_not_merge_across_partitions_select_final>\n <async_insert>1</async_insert>\n <wait_for_async_insert>1</wait_for_async_insert>\n <async_insert_max_data_size>10000000</async_insert_max_data_size>\n <!-- Adaptive batching: tune flush window between 50 ms (low-latency rare inserts)\n and 2 s (absorbs bursts). A single fixed value is silently overridden by the\n adaptive algorithm. -->\n <async_insert_busy_timeout_min_ms>50</async_insert_busy_timeout_min_ms>\n <async_insert_busy_timeout_max_ms>2000</async_insert_busy_timeout_max_ms>\n <async_insert_use_adaptive_busy_timeout>1</async_insert_use_adaptive_busy_timeout>\n <!-- Server-side deduplication of async inserts. Latent retry safety net:\n if a producer retries the same insert window (network hiccup, lambda re-run,\n SQS redelivery), the second attempt collapses against the first. As of CH 26.1\n this also propagates end-to-end through dependent materialised views \u2014 without\n it, a retried insert could double-count in metrics_hourly_mv / log_severity_hourly_mv\n even if the base table dedups. CH pin is 26.3 so the propagation fix is in. -->\n <async_insert_deduplicate>1</async_insert_deduplicate>\n <input_format_parallel_parsing>0</input_format_parallel_parsing>\n <output_format_parallel_formatting>0</output_format_parallel_formatting>\n <!-- Lazy materialisation (CH 25.4+): for `SELECT * ... LIMIT N` shapes the planner\n reads only the columns needed to evaluate ORDER BY / WHERE, then fetches the\n remaining columns for the surviving N rows. Order-of-magnitude I/O reduction\n on dashboard queries (e.g. getLatestMetrics LIMIT 1 BY application_id). -->\n <query_plan_optimize_lazy_materialization>1</query_plan_optimize_lazy_materialization>\n <!-- Per-query memory cap (overrides server-wide max_memory_usage of 1 GB\n to give app_writer 2 GB headroom). Belt-and-braces with the inline\n SETTINGS in 002-users.sql so neither layer can drift alone. -->\n <max_memory_usage>2000000000</max_memory_usage>\n <max_memory_usage_for_user>2684354560</max_memory_usage_for_user>\n <max_bytes_before_external_sort>536870912</max_bytes_before_external_sort>\n <max_bytes_before_external_group_by>536870912</max_bytes_before_external_group_by>\n <!-- Per-query caps. Belt-and-braces with the inline SETTINGS in\n 002-users.sql so `CREATE OR REPLACE USER` cannot regress the bound. -->\n <max_execution_time>30</max_execution_time>\n <max_rows_to_read>10000000</max_rows_to_read>\n </app_writer>\n <audit_writer>\n <max_threads>1</max_threads>\n <max_insert_threads>1</max_insert_threads>\n <max_concurrent_queries_for_user>2</max_concurrent_queries_for_user>\n <max_memory_usage>500000000</max_memory_usage>\n <max_execution_time>10</max_execution_time>\n <async_insert>1</async_insert>\n <wait_for_async_insert>1</wait_for_async_insert>\n </audit_writer>\n <backup_reader>\n <max_threads>2</max_threads>\n <max_concurrent_queries_for_user>1</max_concurrent_queries_for_user>\n <max_memory_usage>1000000000</max_memory_usage>\n <max_execution_time>3600</max_execution_time>\n </backup_reader>\n <schema_admin>\n <max_threads>2</max_threads>\n <max_concurrent_queries_for_user>1</max_concurrent_queries_for_user>\n <max_memory_usage>1000000000</max_memory_usage>\n <max_execution_time>1800</max_execution_time>\n </schema_admin>\n </profiles>\n <quotas>\n <tenant_default>\n <interval>\n <duration>3600</duration>\n <queries>1000</queries>\n <result_rows>10000000</result_rows>\n </interval>\n </tenant_default>\n </quotas>\n</clickhouse>";
6
- export declare function generateClickHouseUserData(options?: ClickHouseUserDataOptions): string;
@@ -1,299 +0,0 @@
1
- import { CLICKHOUSE_DATA_MOUNT_PATH, CLICKHOUSE_EBS_DEVICE_NAME, CLICKHOUSE_CONFIG_SUBDIR, CLICKHOUSE_USERS_SUBDIR, CLICKHOUSE_HTTP_PORT, CLICKHOUSE_PROMETHEUS_PORT } from "./clickhouseConstants.js";
2
- function generateServerConfigXml(cfAccountId) {
3
- if (cfAccountId !== undefined && !/^[a-f0-9]{32}$/.test(cfAccountId)) {
4
- throw new Error(`Invalid Cloudflare account ID format: expected 32-char hex, got "${cfAccountId}"`);
5
- }
6
- const storageBlock = cfAccountId !== undefined
7
- ? ` <storage_configuration>
8
- <disks>
9
- <local_ssd>
10
- <type>local</type>
11
- <path>/var/lib/clickhouse/</path>
12
- </local_ssd>
13
- <r2_cold>
14
- <type>s3</type>
15
- <endpoint>https://${cfAccountId}.r2.cloudflarestorage.com/fjall-clickhouse-cold/data/</endpoint>
16
- <access_key_id from_env="R2_ACCESS_KEY" />
17
- <secret_access_key from_env="R2_SECRET_KEY" />
18
- <region>auto</region>
19
- <support_batch_delete>false</support_batch_delete>
20
- <metadata_path>/var/lib/clickhouse/disks/r2_cold/</metadata_path>
21
- </r2_cold>
22
- </disks>
23
- <policies>
24
- <tiered>
25
- <volumes>
26
- <hot>
27
- <disk>local_ssd</disk>
28
- </hot>
29
- <cold>
30
- <disk>r2_cold</disk>
31
- </cold>
32
- </volumes>
33
- <move_factor>0.05</move_factor>
34
- </tiered>
35
- <local_only>
36
- <volumes>
37
- <default>
38
- <disk>local_ssd</disk>
39
- </default>
40
- </volumes>
41
- </local_only>
42
- </policies>
43
- </storage_configuration>`
44
- : ` <storage_configuration>
45
- <disks>
46
- <default>
47
- <keep_free_space_bytes>1073741824</keep_free_space_bytes>
48
- </default>
49
- </disks>
50
- <policies>
51
- <tiered>
52
- <volumes>
53
- <hot>
54
- <disk>default</disk>
55
- </hot>
56
- <cold>
57
- <disk>default</disk>
58
- </cold>
59
- </volumes>
60
- </tiered>
61
- <local_only>
62
- <volumes>
63
- <default>
64
- <disk>default</disk>
65
- </default>
66
- </volumes>
67
- </local_only>
68
- </policies>
69
- </storage_configuration>`;
70
- return `<clickhouse>
71
- <logger>
72
- <level>warning</level>
73
- <log>/var/log/clickhouse-server/clickhouse-server.log</log>
74
- <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
75
- <size>100M</size>
76
- <count>3</count>
77
- </logger>
78
- <max_server_memory_usage>2684354560</max_server_memory_usage>
79
- <max_memory_usage>1000000000</max_memory_usage>
80
- <max_bytes_before_external_sort>536870912</max_bytes_before_external_sort>
81
- <max_bytes_before_external_group_by>536870912</max_bytes_before_external_group_by>
82
- <mark_cache_size>268435456</mark_cache_size>
83
- <index_mark_cache_size>67108864</index_mark_cache_size>
84
- <uncompressed_cache_size>0</uncompressed_cache_size>
85
- <max_concurrent_queries>10</max_concurrent_queries>
86
- <background_pool_size>2</background_pool_size>
87
- <background_schedule_pool_size>2</background_schedule_pool_size>
88
- <background_merges_mutations_concurrency_ratio>2</background_merges_mutations_concurrency_ratio>
89
- <background_move_pool_size>1</background_move_pool_size>
90
- <background_fetches_pool_size>1</background_fetches_pool_size>
91
- <background_message_broker_schedule_pool_size>1</background_message_broker_schedule_pool_size>
92
- <merge_tree>
93
- <max_suspicious_broken_parts>5</max_suspicious_broken_parts>
94
- <parts_to_delay_insert>150</parts_to_delay_insert>
95
- <parts_to_throw_insert>300</parts_to_throw_insert>
96
- </merge_tree>
97
- <http_port>${CLICKHOUSE_HTTP_PORT}</http_port>
98
- <custom_settings_prefixes>current_</custom_settings_prefixes>
99
- <!-- HTTP keep-alive window. Must exceed @clickhouse/client idle_socket_ttl (15 s)
100
- so the client always closes the socket first. Prevents ECONNRESET on reuse. -->
101
- <keep_alive_timeout>30</keep_alive_timeout>
102
- <query_log>
103
- <database>system</database>
104
- <table>query_log</table>
105
- <flush_interval_milliseconds>7500</flush_interval_milliseconds>
106
- <ttl>event_date + INTERVAL 14 DAY DELETE</ttl>
107
- </query_log>
108
- <!-- All system log tables need bounded retention on 80GB EBS (mirror query_log's 14-day policy). -->
109
- <asynchronous_metric_log>
110
- <database>system</database>
111
- <table>asynchronous_metric_log</table>
112
- <ttl>event_date + INTERVAL 14 DAY DELETE</ttl>
113
- </asynchronous_metric_log>
114
- <metric_log>
115
- <database>system</database>
116
- <table>metric_log</table>
117
- <ttl>event_date + INTERVAL 14 DAY DELETE</ttl>
118
- </metric_log>
119
- <part_log>
120
- <database>system</database>
121
- <table>part_log</table>
122
- <ttl>event_date + INTERVAL 14 DAY DELETE</ttl>
123
- </part_log>
124
- <error_log>
125
- <database>system</database>
126
- <table>error_log</table>
127
- <ttl>event_date + INTERVAL 14 DAY DELETE</ttl>
128
- </error_log>
129
- ${storageBlock}
130
- <query_cache>
131
- <max_size_in_bytes>104857600</max_size_in_bytes>
132
- <max_entries>1024</max_entries>
133
- <max_entry_size_in_bytes>1048576</max_entry_size_in_bytes>
134
- </query_cache>
135
- <prometheus>
136
- <endpoint>/metrics</endpoint>
137
- <port>${CLICKHOUSE_PROMETHEUS_PORT}</port>
138
- <metrics>true</metrics>
139
- <events>true</events>
140
- <asynchronous_metrics>true</asynchronous_metrics>
141
- </prometheus>
142
- <query_thread_log remove="1"/>
143
- <opentelemetry_span_log remove="1"/>
144
- <processors_profile_log remove="1"/>
145
- </clickhouse>`;
146
- }
147
- export const USERS_CONFIG_XML = `<clickhouse>
148
- <users>
149
- <default>
150
- <networks>
151
- <ip>127.0.0.1</ip>
152
- <ip>::1</ip>
153
- </networks>
154
- </default>
155
- </users>
156
- <profiles>
157
- <default>
158
- <optimize_move_to_prewhere>1</optimize_move_to_prewhere>
159
- <!-- ALTER TABLE ... MODIFY TTL on a 30-day-partitioned table would otherwise
160
- trigger an immediate full-table rewrite (default = 1). On the t4g.medium
161
- box that's a merge-pool starvation event. Keep TTL changes lazy: parts
162
- re-evaluate TTL on their next natural merge, no forced rewrite. -->
163
- <materialize_ttl_after_modify>0</materialize_ttl_after_modify>
164
- </default>
165
- <app_writer>
166
- <max_threads>2</max_threads>
167
- <max_insert_threads>1</max_insert_threads>
168
- <max_concurrent_queries_for_user>4</max_concurrent_queries_for_user>
169
- <log_queries_min_query_duration_ms>100</log_queries_min_query_duration_ms>
170
- <optimize_move_to_prewhere>1</optimize_move_to_prewhere>
171
- <use_query_condition_cache>1</use_query_condition_cache>
172
- <!-- Re-enable skip indexes under FINAL (tenantQuery auto-FINALs RMT tables;
173
- default disables idx_aws_account, idx_application, idx_dedup, idx_fingerprint). -->
174
- <use_skip_indexes_if_final>1</use_skip_indexes_if_final>
175
- <!-- Tenant-isolation guards (ClickHouse PR #91065 fix). Belt-and-braces with the
176
- per-user SQL SETTINGS in 002-users.sql — keep both so \`CREATE OR REPLACE USER\`
177
- cannot regress this. Without these flags, FINAL queries on un-merged
178
- ReplacingMergeTree parts can leak across tenants. -->
179
- <apply_row_policy_after_final>1</apply_row_policy_after_final>
180
- <apply_prewhere_after_final>1</apply_prewhere_after_final>
181
- <do_not_merge_across_partitions_select_final>1</do_not_merge_across_partitions_select_final>
182
- <async_insert>1</async_insert>
183
- <wait_for_async_insert>1</wait_for_async_insert>
184
- <async_insert_max_data_size>10000000</async_insert_max_data_size>
185
- <!-- Adaptive batching: tune flush window between 50 ms (low-latency rare inserts)
186
- and 2 s (absorbs bursts). A single fixed value is silently overridden by the
187
- adaptive algorithm. -->
188
- <async_insert_busy_timeout_min_ms>50</async_insert_busy_timeout_min_ms>
189
- <async_insert_busy_timeout_max_ms>2000</async_insert_busy_timeout_max_ms>
190
- <async_insert_use_adaptive_busy_timeout>1</async_insert_use_adaptive_busy_timeout>
191
- <!-- Server-side deduplication of async inserts. Latent retry safety net:
192
- if a producer retries the same insert window (network hiccup, lambda re-run,
193
- SQS redelivery), the second attempt collapses against the first. As of CH 26.1
194
- this also propagates end-to-end through dependent materialised views — without
195
- it, a retried insert could double-count in metrics_hourly_mv / log_severity_hourly_mv
196
- even if the base table dedups. CH pin is 26.3 so the propagation fix is in. -->
197
- <async_insert_deduplicate>1</async_insert_deduplicate>
198
- <input_format_parallel_parsing>0</input_format_parallel_parsing>
199
- <output_format_parallel_formatting>0</output_format_parallel_formatting>
200
- <!-- Lazy materialisation (CH 25.4+): for \`SELECT * ... LIMIT N\` shapes the planner
201
- reads only the columns needed to evaluate ORDER BY / WHERE, then fetches the
202
- remaining columns for the surviving N rows. Order-of-magnitude I/O reduction
203
- on dashboard queries (e.g. getLatestMetrics LIMIT 1 BY application_id). -->
204
- <query_plan_optimize_lazy_materialization>1</query_plan_optimize_lazy_materialization>
205
- <!-- Per-query memory cap (overrides server-wide max_memory_usage of 1 GB
206
- to give app_writer 2 GB headroom). Belt-and-braces with the inline
207
- SETTINGS in 002-users.sql so neither layer can drift alone. -->
208
- <max_memory_usage>2000000000</max_memory_usage>
209
- <max_memory_usage_for_user>2684354560</max_memory_usage_for_user>
210
- <max_bytes_before_external_sort>536870912</max_bytes_before_external_sort>
211
- <max_bytes_before_external_group_by>536870912</max_bytes_before_external_group_by>
212
- <!-- Per-query caps. Belt-and-braces with the inline SETTINGS in
213
- 002-users.sql so \`CREATE OR REPLACE USER\` cannot regress the bound. -->
214
- <max_execution_time>30</max_execution_time>
215
- <max_rows_to_read>10000000</max_rows_to_read>
216
- </app_writer>
217
- <audit_writer>
218
- <max_threads>1</max_threads>
219
- <max_insert_threads>1</max_insert_threads>
220
- <max_concurrent_queries_for_user>2</max_concurrent_queries_for_user>
221
- <max_memory_usage>500000000</max_memory_usage>
222
- <max_execution_time>10</max_execution_time>
223
- <async_insert>1</async_insert>
224
- <wait_for_async_insert>1</wait_for_async_insert>
225
- </audit_writer>
226
- <backup_reader>
227
- <max_threads>2</max_threads>
228
- <max_concurrent_queries_for_user>1</max_concurrent_queries_for_user>
229
- <max_memory_usage>1000000000</max_memory_usage>
230
- <max_execution_time>3600</max_execution_time>
231
- </backup_reader>
232
- <schema_admin>
233
- <max_threads>2</max_threads>
234
- <max_concurrent_queries_for_user>1</max_concurrent_queries_for_user>
235
- <max_memory_usage>1000000000</max_memory_usage>
236
- <max_execution_time>1800</max_execution_time>
237
- </schema_admin>
238
- </profiles>
239
- <quotas>
240
- <tenant_default>
241
- <interval>
242
- <duration>3600</duration>
243
- <queries>1000</queries>
244
- <result_rows>10000000</result_rows>
245
- </interval>
246
- </tenant_default>
247
- </quotas>
248
- </clickhouse>`;
249
- export function generateClickHouseUserData(options) {
250
- const serverConfigXml = generateServerConfigXml(options?.cfAccountId);
251
- return `#!/bin/bash
252
- set -euo pipefail
253
-
254
- DEVICE="${CLICKHOUSE_EBS_DEVICE_NAME}"
255
- MOUNT_POINT="${CLICKHOUSE_DATA_MOUNT_PATH}"
256
-
257
- # Wait for the EBS volume to be attached (up to 60 seconds)
258
- for i in $(seq 1 60); do
259
- if [ -b "$DEVICE" ]; then
260
- break
261
- fi
262
- sleep 1
263
- done
264
-
265
- if [ ! -b "$DEVICE" ]; then
266
- echo "ERROR: EBS volume $DEVICE not found after 60 seconds" >&2
267
- exit 1
268
- fi
269
-
270
- # Format only if not already formatted (idempotent)
271
- if ! blkid "$DEVICE" | grep -q ext4; then
272
- mkfs.ext4 -m 0 "$DEVICE"
273
- fi
274
-
275
- # Mount
276
- mkdir -p "$MOUNT_POINT"
277
- mount "$DEVICE" "$MOUNT_POINT"
278
-
279
- # Persist across reboots
280
- if ! grep -q "$MOUNT_POINT" /etc/fstab; then
281
- echo "$DEVICE $MOUNT_POINT ext4 defaults,nofail 0 2" >> /etc/fstab
282
- fi
283
-
284
- # Write ClickHouse server config (memory limits, storage policies, Prometheus)
285
- mkdir -p "$MOUNT_POINT/${CLICKHOUSE_CONFIG_SUBDIR}"
286
- cat > "$MOUNT_POINT/${CLICKHOUSE_CONFIG_SUBDIR}/fjall.xml" << 'CONFIGEOF'
287
- ${serverConfigXml}
288
- CONFIGEOF
289
-
290
- # Write ClickHouse users config (profiles, quotas, default user restrictions)
291
- mkdir -p "$MOUNT_POINT/${CLICKHOUSE_USERS_SUBDIR}"
292
- cat > "$MOUNT_POINT/${CLICKHOUSE_USERS_SUBDIR}/fjall.xml" << 'USERSEOF'
293
- ${USERS_CONFIG_XML}
294
- USERSEOF
295
-
296
- # ClickHouse alpine image runs as uid 101, gid 101
297
- chown -R 101:101 "$MOUNT_POINT"
298
- `;
299
- }
@@ -1,4 +0,0 @@
1
- export { default as ClickHouse } from "./clickhouse.js";
2
- export type { ClickHouseProps, ClickHouseR2Config, ClickHouseOutputs } from "./clickhouseTypes.js";
3
- export { createClickHouseAlarms } from "./clickhouseAlarms.js";
4
- export type { ClickHouseAlarmThresholds, ClickHouseAlarmsProps } from "./clickhouseAlarms.js";
@@ -1,2 +0,0 @@
1
- export { default as ClickHouse } from "./clickhouse.js";
2
- export { createClickHouseAlarms } from "./clickhouseAlarms.js";
@@ -1,2 +0,0 @@
1
- import { AutoScalingGroup } from "aws-cdk-lib/aws-autoscaling";
2
- export declare function regression(asg: AutoScalingGroup, queue: any, id: string): void;
@@ -1,11 +0,0 @@
1
- import { DefaultResult, LifecycleTransition } from "aws-cdk-lib/aws-autoscaling";
2
- import { QueueHook } from "aws-cdk-lib/aws-autoscaling-hooktargets";
3
- import { Duration } from "aws-cdk-lib";
4
- export function regression(asg, queue, id) {
5
- asg.addLifecycleHook(`${id}LaunchingHook`, {
6
- lifecycleTransition: LifecycleTransition.INSTANCE_LAUNCHING,
7
- defaultResult: DefaultResult.ABANDON,
8
- heartbeatTimeout: Duration.seconds(300),
9
- notificationTarget: new QueueHook(queue)
10
- });
11
- }
@@ -1,7 +0,0 @@
1
- import { CfnOutput } from "aws-cdk-lib";
2
- import { Construct } from "constructs";
3
- export declare class DefaultEventBus extends Construct {
4
- readonly defaultEventBusName: CfnOutput;
5
- readonly defaultEventBusArn: CfnOutput;
6
- constructor(scope: Construct, id: string);
7
- }
@@ -1,21 +0,0 @@
1
- import { CfnOutput } from "aws-cdk-lib";
2
- import { EventBus } from "aws-cdk-lib/aws-events";
3
- import { Construct } from "constructs";
4
- export class DefaultEventBus extends Construct {
5
- defaultEventBusName;
6
- defaultEventBusArn;
7
- constructor(scope, id) {
8
- super(scope, id);
9
- const eventBridge = EventBus.fromEventBusName(this, "DefaultEventBus", "default");
10
- this.defaultEventBusName = new CfnOutput(this, "DefaultEventBusNameOutput", {
11
- key: "DefaultEventBusName",
12
- value: eventBridge.eventBusName,
13
- exportName: "DefaultEventBusName"
14
- });
15
- this.defaultEventBusArn = new CfnOutput(this, "DefaultEventBusArnOutput", {
16
- key: "DefaultEventBusArn",
17
- value: eventBridge.eventBusArn,
18
- exportName: "DefaultEventBusArn"
19
- });
20
- }
21
- }
@@ -1,13 +0,0 @@
1
- import { Construct } from "constructs";
2
- import { type IHostedZone } from "aws-cdk-lib/aws-route53";
3
- import type { DomainApexProps } from "../../../utils/domainTypes.js";
4
- export declare class Domain extends Construct {
5
- readonly hostedZoneId: string;
6
- readonly hostedZone: IHostedZone;
7
- private readonly zoneName;
8
- constructor(scope: Construct, id: string, props: DomainApexProps);
9
- private addDelegationRole;
10
- private addZoneIdOutput;
11
- private addNameserverOutput;
12
- private addCertificates;
13
- }
@@ -1,100 +0,0 @@
1
- import { Construct } from "constructs";
2
- import { CfnOutput, Fn } from "aws-cdk-lib";
3
- import { toPascalCase, getSafeZoneName } from "../../../utils/capitaliseString.js";
4
- import { Role } from "../iam/index.js";
5
- import { OrganizationPrincipal, PolicyDocument, PolicyStatement } from "aws-cdk-lib/aws-iam";
6
- import { HostedZone as AWSHostedZone } from "aws-cdk-lib/aws-route53";
7
- import { getDomainExportNames, addDnsRecords } from "../../../utils/domainTypes.js";
8
- import { DomainCertificate } from "./domainCertificate.js";
9
- export class Domain extends Construct {
10
- hostedZoneId;
11
- hostedZone;
12
- zoneName;
13
- constructor(scope, id, props) {
14
- super(scope, id);
15
- this.zoneName = props.zoneName;
16
- if (!props.hostedZoneId) {
17
- const createdZone = new AWSHostedZone(this, `${getSafeZoneName(props.zoneName)}HostedZone`, {
18
- zoneName: props.zoneName,
19
- comment: `Hosted Zone for ${props.zoneName}`
20
- });
21
- this.hostedZone = createdZone;
22
- this.hostedZoneId = createdZone.hostedZoneId;
23
- this.addDelegationRole(createdZone);
24
- this.addNameserverOutput(createdZone);
25
- }
26
- else {
27
- this.hostedZoneId = props.hostedZoneId;
28
- this.hostedZone = AWSHostedZone.fromHostedZoneAttributes(this, `${getSafeZoneName(props.zoneName)}ImportedHostedZone`, {
29
- hostedZoneId: props.hostedZoneId,
30
- zoneName: props.zoneName
31
- });
32
- }
33
- this.addZoneIdOutput();
34
- if (props.records) {
35
- addDnsRecords(this, this.hostedZone, this.zoneName, props.records);
36
- }
37
- if (props.certificates) {
38
- this.addCertificates(props.certificates);
39
- }
40
- }
41
- addDelegationRole(zone) {
42
- const domainLabel = this.zoneName.split(".")[0] ?? "default";
43
- const safeDomainLabel = toPascalCase(domainLabel);
44
- const role = new Role(this, `${safeDomainLabel}DelegateHostedZoneRole`, {
45
- assumedBy: new OrganizationPrincipal(Fn.importValue("OrganisationId")),
46
- roleName: `${domainLabel}DelegateHostedZoneRole`,
47
- inlinePolicies: {
48
- ["listHostedZones"]: new PolicyDocument({
49
- statements: [
50
- new PolicyStatement({
51
- actions: ["route53:ListHostedZonesByName"],
52
- resources: ["*"]
53
- })
54
- ]
55
- }),
56
- ["changeResourceRecordSets"]: new PolicyDocument({
57
- statements: [
58
- new PolicyStatement({
59
- actions: ["route53:ChangeResourceRecordSets"],
60
- resources: [`arn:aws:route53:::hostedzone/${zone.hostedZoneId}`]
61
- })
62
- ]
63
- })
64
- }
65
- });
66
- zone.grantDelegation(role);
67
- new CfnOutput(this, `${safeDomainLabel}DelegateHostedZoneRoleArn`, {
68
- key: `${safeDomainLabel}DelegateHostedZoneRoleArn`,
69
- value: role.roleArn,
70
- exportName: `${domainLabel}DelegateHostedZoneRoleArn`
71
- });
72
- }
73
- addZoneIdOutput() {
74
- const safeKey = toPascalCase(getSafeZoneName(this.zoneName));
75
- const exports = getDomainExportNames(this.zoneName);
76
- new CfnOutput(this, `${safeKey}HostedZoneId`, {
77
- key: `${safeKey}HostedZoneId`,
78
- value: this.hostedZoneId,
79
- exportName: exports.hostedZoneId
80
- });
81
- }
82
- addNameserverOutput(zone) {
83
- const safeKey = toPascalCase(getSafeZoneName(this.zoneName));
84
- new CfnOutput(this, `${safeKey}Nameservers`, {
85
- key: `${safeKey}Nameservers`,
86
- value: Fn.join(",", zone.hostedZoneNameServers ?? [])
87
- });
88
- }
89
- addCertificates(certificates) {
90
- const safeZone = toPascalCase(getSafeZoneName(this.zoneName));
91
- certificates.forEach((cert, index) => {
92
- const safeCertName = toPascalCase(cert.domainName.split(".").join(""));
93
- new DomainCertificate(this, `${safeZone}${safeCertName}Cert${index}`, {
94
- domainName: cert.domainName,
95
- subjectAlternativeNames: cert.subjectAlternativeNames,
96
- hostedZone: this.hostedZone
97
- });
98
- });
99
- }
100
- }
@@ -1 +0,0 @@
1
- export {};
@@ -1,42 +0,0 @@
1
- import { App, Stack } from "aws-cdk-lib";
2
- import { Template } from "aws-cdk-lib/assertions";
3
- import { Vpc } from "aws-cdk-lib/aws-ec2";
4
- import { Repository } from "aws-cdk-lib/aws-ecr";
5
- import { mkdtempSync, mkdirSync } from "node:fs";
6
- import { tmpdir } from "node:os";
7
- import { join } from "node:path";
8
- import { EcsCompute } from "./patterns/aws/computeEcs.js";
9
- import { RelationalDatabase } from "./patterns/aws/database.js";
10
- const root = mkdtempSync(join(tmpdir(), "ds-"));
11
- mkdirSync(join(root, "20260512000000_init"));
12
- const app = new App();
13
- const stack = new Stack(app, "S", { env: { account: "123456789012", region: "us-east-1" } });
14
- const vpc = new Vpc(stack, "Vpc");
15
- const db = new RelationalDatabase(stack, "Db", {
16
- type: "Instance", vpc, databaseName: "appdata",
17
- migrations: { tool: "prisma", dir: root }
18
- });
19
- new EcsCompute(stack, "Cluster", {
20
- type: "ecs",
21
- ecrRepository: new Repository(stack, "Ecr"),
22
- services: [{
23
- name: "web", capacityProvider: "FARGATE",
24
- containers: [{ name: "app", image: "nginx:latest", port: 3000 }],
25
- connections: [db],
26
- migrations: {
27
- mode: "lifecycle-hook", command: ["node", "migrate.mjs"], entryPoint: ["/usr/bin/tini", "--"],
28
- separateTaskDef: { cpu: 512, memoryLimitMiB: 1024 }
29
- }
30
- }]
31
- });
32
- const t = Template.fromStack(stack);
33
- const ing = t.findResources("AWS::EC2::SecurityGroupIngress");
34
- for (const [name, res] of Object.entries(ing)) {
35
- console.log("INGRESS:", name);
36
- console.log(" props:", JSON.stringify(res.Properties));
37
- }
38
- const eg = t.findResources("AWS::EC2::SecurityGroupEgress");
39
- for (const [name, res] of Object.entries(eg)) {
40
- console.log("EGRESS:", name);
41
- console.log(" props:", JSON.stringify(res.Properties));
42
- }
@@ -1,10 +0,0 @@
1
- import { type IVpc } from "aws-cdk-lib/aws-ec2";
2
- import { Ec2Instance } from "../resources/aws/compute/ec2.js";
3
- import { type AwsStack } from "../resources/index.js";
4
- export interface BastionConfig {
5
- instanceType?: string;
6
- }
7
- export interface BastionResult {
8
- bastion: Ec2Instance;
9
- }
10
- export declare function createBastion(networkStack: AwsStack, appName: string, stackPrefix: string, vpc: IVpc, config: BastionConfig | true): BastionResult;