@fjall/components-infrastructure 16.0.1 → 17.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/patterns/aws/clickhouseDatabase.d.ts +103 -8
- package/dist/lib/patterns/aws/clickhouseDatabase.js +51 -12
- package/dist/lib/patterns/aws/database.d.ts +1 -1
- package/dist/lib/patterns/aws/database.js +1 -1
- package/dist/lib/resources/aws/database/clickhouseBackupScript.d.ts +103 -0
- package/dist/lib/resources/aws/database/clickhouseBackupScript.js +123 -0
- package/dist/lib/resources/aws/database/clickhouseConstants.d.ts +132 -20
- package/dist/lib/resources/aws/database/clickhouseConstants.js +126 -15
- package/dist/lib/resources/aws/database/clickhouseSchemas.d.ts +33 -1
- package/dist/lib/resources/aws/database/clickhouseSchemas.js +31 -0
- package/dist/lib/resources/aws/database/clickhouseStorage.d.ts +58 -0
- package/dist/lib/resources/aws/database/clickhouseStorage.js +87 -0
- package/dist/lib/resources/aws/database/clickhouseTuning.d.ts +5 -6
- package/dist/lib/resources/aws/database/clickhouseTuning.js +39 -16
- package/dist/lib/resources/aws/database/clickhouseUserData.d.ts +11 -0
- package/dist/lib/resources/aws/database/clickhouseUserData.js +22 -6
- package/dist/lib/resources/aws/database/clickhouseXmlRenderer.js +22 -1
- package/dist/lib/resources/aws/monitoring/clickhouseAlarms.d.ts +54 -5
- package/dist/lib/resources/aws/monitoring/clickhouseAlarms.js +106 -10
- package/package.json +3 -3
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { ClickHouseInstanceType } from "@fjall/util/clickhouse";
|
|
1
2
|
/** Database name created at ClickHouse bootstrap; consumed by BACKUP DATABASE,
|
|
2
3
|
* OPTIMIZE TABLE, and the DatabaseName CfnOutput so all four sites share one source. */
|
|
3
4
|
export declare const CLICKHOUSE_DATABASE_NAME = "analytics";
|
|
@@ -42,28 +43,67 @@ export declare const DEFAULT_CLICKHOUSE_INSTANCE_TYPE = "m7g.medium";
|
|
|
42
43
|
* that pulls once per launch. Upstream CH CI runs its full perf + stress
|
|
43
44
|
* matrix on the Ubuntu build; Alpine is community-tier coverage. */
|
|
44
45
|
export declare const CLICKHOUSE_IMAGE = "docker.io/clickhouse/clickhouse-server:26.3.17.56";
|
|
46
|
+
/** Absolute floor of the disk-critical alarm, in GiB.
|
|
47
|
+
*
|
|
48
|
+
* A pure percentage cannot express "about to break": 85 % used is 12 GiB
|
|
49
|
+
* free on an 80 GiB volume and 600 GiB free on a 4 TiB one, and only one of
|
|
50
|
+
* those is an incident. But a pure constant cannot either, because the other
|
|
51
|
+
* half of the failure scales with the data — ClickHouse will not select a
|
|
52
|
+
* merge whose source parts need more than the free space allows, and parts
|
|
53
|
+
* grow with the table. So the default floor is the LARGER of this constant
|
|
54
|
+
* and `CLICKHOUSE_DISK_FREE_CRITICAL_SHARE` of the volume: the constant
|
|
55
|
+
* guards response time on a small disk, the share guards merge headroom on a
|
|
56
|
+
* large one. See `resolveClickHouseStorage`.
|
|
57
|
+
*
|
|
58
|
+
* 20 GiB is days of ingest headroom at the observed rate — enough lead time
|
|
59
|
+
* to grow the volume, which takes an instance restart and cannot be done in
|
|
60
|
+
* minutes. */
|
|
61
|
+
export declare const CLICKHOUSE_DISK_FREE_CRITICAL_GIB = 20;
|
|
62
|
+
/** Share of the volume the disk-critical floor rises to on larger disks.
|
|
63
|
+
* At 5 %, a 4 TiB volume pages with 200 GiB free — roughly twice a large
|
|
64
|
+
* part, so merges still have room to complete while the operator responds.
|
|
65
|
+
* The two halves cross over at 400 GiB, the default volume size. */
|
|
66
|
+
export declare const CLICKHOUSE_DISK_FREE_CRITICAL_SHARE = 0.05;
|
|
45
67
|
/** EBS volume configuration.
|
|
46
68
|
*
|
|
47
|
-
*
|
|
48
|
-
* not
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
69
|
+
* `CLICKHOUSE_EBS_VOLUME_SIZE_GB` is the DEFAULT for
|
|
70
|
+
* `ClickHouseDatabaseProps.storageGb`, not a fixed size. 400 GiB is sized
|
|
71
|
+
* for a mid-size multi-tenant analytics workload — order of tens of tenant
|
|
72
|
+
* organisations each carrying full `log_events` / `application_metrics`
|
|
73
|
+
* retention windows — with enough headroom that the first resize is a
|
|
74
|
+
* deliberate capacity decision rather than an early surprise. (The
|
|
75
|
+
* reference deployment's history is the evidence: 80 GiB bound at roughly
|
|
76
|
+
* two-thirds of that scale.) gp3 storage is cheap relative to an
|
|
77
|
+
* out-of-disk ClickHouse — merges stop, inserts start failing, and the
|
|
78
|
+
* recovery is a restore rather than a resize. Workloads outside that
|
|
79
|
+
* profile set `storageGb` directly, exactly as they would size an RDS
|
|
80
|
+
* volume.
|
|
81
|
+
*
|
|
82
|
+
* Raising the size alone is not enough — `buildClickHouseUserData` must grow
|
|
83
|
+
* the ext4 filesystem to match, because ModifyVolume resizes the block
|
|
84
|
+
* device and nothing else. The pair moves together: the declared size is
|
|
85
|
+
* embedded in the user data, so a `storageGb` change versions the launch
|
|
86
|
+
* template and triggers the instance refresh whose boot runs `resize2fs`.
|
|
87
|
+
* EBS permits one modification per volume per 6 hours.
|
|
88
|
+
*
|
|
89
|
+
* IOPS and throughput default to the gp3 baseline — the figures the volume
|
|
90
|
+
* price already includes — and are the construct's `iops` / `throughputMbps`
|
|
91
|
+
* props, the way RDS exposes provisioned IOPS and throughput on gp3. They are
|
|
92
|
+
* size-independent on gp3 and the default workload is merge- and scan-bound
|
|
93
|
+
* rather than IOPS-starved, so the baseline is the right default; a caller
|
|
94
|
+
* whose scans are bandwidth-bound raises them, within the bounds and ratios
|
|
95
|
+
* `resolveClickHouseVolumePerformance` enforces. */
|
|
64
96
|
export declare const CLICKHOUSE_EBS_VOLUME_SIZE_GB = 400;
|
|
65
97
|
export declare const CLICKHOUSE_EBS_IOPS = 3000;
|
|
66
98
|
export declare const CLICKHOUSE_EBS_THROUGHPUT_MBPS = 125;
|
|
99
|
+
/** Volume and backup-retention bounds. Re-exported, NOT redeclared:
|
|
100
|
+
* `@fjall/generator` enforces the same numbers when it scaffolds or rewrites
|
|
101
|
+
* an `infrastructure.ts` declaration, and a second copy here is exactly the
|
|
102
|
+
* drift `backupRetentionDays` shipped for as long as it borrowed the RDS
|
|
103
|
+
* schema (generator 35 vs construct 3650). One source, two enforcement
|
|
104
|
+
* sites. */
|
|
105
|
+
export { CLICKHOUSE_MIN_STORAGE_GB, CLICKHOUSE_MAX_STORAGE_GB, CLICKHOUSE_MIN_IOPS, CLICKHOUSE_MAX_IOPS, CLICKHOUSE_MAX_IOPS_PER_GB, CLICKHOUSE_MIN_THROUGHPUT_MBPS, CLICKHOUSE_MAX_THROUGHPUT_MBPS, CLICKHOUSE_IOPS_PER_THROUGHPUT_MBPS, CLICKHOUSE_MIN_BACKUP_RETENTION_DAYS, CLICKHOUSE_MAX_BACKUP_RETENTION_DAYS, CLICKHOUSE_INSTANCE_TYPES } from "@fjall/util/clickhouse";
|
|
106
|
+
export type { ClickHouseInstanceType } from "@fjall/util/clickhouse";
|
|
67
107
|
/** Host memory reserved from the ClickHouse container: kernel + ECS agent +
|
|
68
108
|
* the host-metrics timer. The ECS agent advertises MemTotal minus
|
|
69
109
|
* CLICKHOUSE_ECS_RESERVED_MEMORY_MIB, and MemTotal itself runs below the
|
|
@@ -81,13 +121,25 @@ export declare const CLICKHOUSE_ECS_RESERVED_MEMORY_MIB = 256;
|
|
|
81
121
|
/** Hardware spec of every instance type the ClickHouse construct supports.
|
|
82
122
|
* Values are the AWS nominal figures. Single source of truth for BOTH the
|
|
83
123
|
* container memory limit (`clickHouseTaskMemoryMiB`) and the derived server
|
|
84
|
-
* tuning + default profiles (`clickhouseTuning.ts`)
|
|
85
|
-
*
|
|
124
|
+
* tuning + default profiles (`clickhouseTuning.ts`).
|
|
125
|
+
*
|
|
126
|
+
* Keyed on `ClickHouseInstanceType` — the supported-type vocabulary lives in
|
|
127
|
+
* `@fjall/util/clickhouse` so the generator can validate `instanceType` at
|
|
128
|
+
* scaffold time against the same list. The `Record` key type makes the
|
|
129
|
+
* parity two-way at compile time: a vocabulary entry without a spec row is a
|
|
130
|
+
* missing-property error, a spec row outside the vocabulary is an
|
|
131
|
+
* excess-property error. Supporting a new type = add it to the tuple there
|
|
132
|
+
* and the spec row here; everything else derives. */
|
|
86
133
|
export interface ClickHouseInstanceSpec {
|
|
87
134
|
vcpus: number;
|
|
88
135
|
memoryGib: number;
|
|
89
136
|
}
|
|
90
|
-
export declare const CLICKHOUSE_INSTANCE_SPECS: Record<
|
|
137
|
+
export declare const CLICKHOUSE_INSTANCE_SPECS: Record<ClickHouseInstanceType, ClickHouseInstanceSpec>;
|
|
138
|
+
/** Guarded string-keyed lookup into `CLICKHOUSE_INSTANCE_SPECS` for
|
|
139
|
+
* caller-supplied types (props carry `string`); `undefined` for unknown
|
|
140
|
+
* types, which every caller converts into its own named synth throw. The
|
|
141
|
+
* exhaustively-keyed Record above stays the compile-time contract. */
|
|
142
|
+
export declare function clickHouseInstanceSpec(instanceType: string): ClickHouseInstanceSpec | undefined;
|
|
91
143
|
/** Derived memory-only view of CLICKHOUSE_INSTANCE_SPECS. */
|
|
92
144
|
export declare const CLICKHOUSE_INSTANCE_MEMORY_GIB: Record<string, number>;
|
|
93
145
|
/** ECS container memory for the ClickHouse server task, derived from the
|
|
@@ -179,6 +231,7 @@ export declare const CLICKHOUSE_HOST_METRICS: {
|
|
|
179
231
|
readonly namespace: "CWAgent";
|
|
180
232
|
readonly memoryMetric: "mem_used_percent";
|
|
181
233
|
readonly diskMetric: "disk_used_percent";
|
|
234
|
+
readonly diskFreeMetric: "disk_free_gib";
|
|
182
235
|
readonly asgDimension: "AutoScalingGroupName";
|
|
183
236
|
};
|
|
184
237
|
/** Shared secret generation options (all ClickHouse users share the same policy). */
|
|
@@ -248,3 +301,62 @@ export declare const BACKUP_TASK_MEMORY_MIB = 256;
|
|
|
248
301
|
export declare const BACKUP_TASK_CPU_UNITS = 256;
|
|
249
302
|
/** Backup object expiration: 14 days (retains 14 daily snapshots). */
|
|
250
303
|
export declare const BACKUP_RETENTION_DAYS = 14;
|
|
304
|
+
/** Scratch database the backup task restores into to prove the backup it just
|
|
305
|
+
* wrote can be read back. Dropped on the way in and on the way out; never
|
|
306
|
+
* queried by the application. */
|
|
307
|
+
export declare const CLICKHOUSE_BACKUP_SCRATCH_DATABASE = "fjall_backup_verify";
|
|
308
|
+
/** Multiple of the critical free-space floor the verify restore must leave
|
|
309
|
+
* clear to proceed.
|
|
310
|
+
*
|
|
311
|
+
* A full restore writes a second copy of the database onto the volume it is
|
|
312
|
+
* verifying, so the check has to be bounded by the same quantity the disk
|
|
313
|
+
* alarm pages on — otherwise the mechanism that protects the data is also
|
|
314
|
+
* the mechanism most likely to fill the disk. Twice the floor rather than
|
|
315
|
+
* once so a verify can never itself be the thing that pages: at exactly the
|
|
316
|
+
* floor the restore would land the volume on the alarm threshold and hold it
|
|
317
|
+
* there until the scratch database is dropped. */
|
|
318
|
+
export declare const CLICKHOUSE_BACKUP_VERIFY_FREE_SPACE_MARGIN = 2;
|
|
319
|
+
/** Backup task log markers. Distinct full tokens, because the CloudWatch
|
|
320
|
+
* metric filters that read them match on substrings — a shared `BACKUP_`
|
|
321
|
+
* stem would make the OK filter count the failures too. */
|
|
322
|
+
export declare const CLICKHOUSE_BACKUP_VERIFY_OK_MARKER = "BACKUP_VERIFY_OK";
|
|
323
|
+
export declare const CLICKHOUSE_BACKUP_VERIFY_FAILED_MARKER = "BACKUP_VERIFY_FAILED";
|
|
324
|
+
export declare const CLICKHOUSE_BACKUP_VERIFY_SKIPPED_MARKER = "BACKUP_VERIFY_SKIPPED_FOR_SIZE";
|
|
325
|
+
/** Bounds on the verify restore's wall-clock ceiling, in seconds.
|
|
326
|
+
*
|
|
327
|
+
* Without a ceiling a hung RESTORE is the one backup failure with NO signal
|
|
328
|
+
* at all: BACKUP_CREATED has already been emitted so the heartbeat is
|
|
329
|
+
* satisfied, and the verify alarms count markers rather than their absence,
|
|
330
|
+
* so a task that never finishes reports nothing. `timeout` turns that
|
|
331
|
+
* silence into exit 124, which takes the same branch as a raised restore
|
|
332
|
+
* and pages.
|
|
333
|
+
*
|
|
334
|
+
* The ceiling scales with the volume (`clickHouseBackupVerifyTimeoutSeconds`
|
|
335
|
+
* below) because the largest restore the free-space gate admits scales with
|
|
336
|
+
* it: on the default 400 GiB the floor of one hour already covers it, but a
|
|
337
|
+
* multi-TiB volume can legitimately need hours, and a fixed hour would page
|
|
338
|
+
* FAILED on every healthy verify past the size where the restore outgrows
|
|
339
|
+
* it. The 20-hour cap keeps a daily-scheduled verify from overlapping its
|
|
340
|
+
* successor — two concurrent runs share the scratch database destructively,
|
|
341
|
+
* so the bound must land inside the schedule interval. */
|
|
342
|
+
export declare const CLICKHOUSE_BACKUP_VERIFY_MIN_TIMEOUT_SECONDS = 3600;
|
|
343
|
+
export declare const CLICKHOUSE_BACKUP_VERIFY_MAX_TIMEOUT_SECONDS = 72000;
|
|
344
|
+
/** Verify-restore timeout for a given data-volume size.
|
|
345
|
+
*
|
|
346
|
+
* Worst case admitted by the free-space gate is a database of roughly half
|
|
347
|
+
* the volume (it must fit beside itself). Writing that back at the gp3
|
|
348
|
+
* baseline throughput (`CLICKHOUSE_EBS_THROUGHPUT_MBPS`) bounds the healthy
|
|
349
|
+
* duration; doubled for read + merge overhead, then clamped to the
|
|
350
|
+
* floor/cap pair above. At the 400 GiB default this resolves to the
|
|
351
|
+
* one-hour floor, so the default behaviour is unchanged from the flat bound
|
|
352
|
+
* it replaced.
|
|
353
|
+
*
|
|
354
|
+
* Deliberately the BASELINE throughput, not the provisioned `throughputMbps`
|
|
355
|
+
* prop. The volume's provisioned figure is an upper bound the instance may
|
|
356
|
+
* not reach — an m7g.medium's EBS bandwidth sits under the gp3 baseline, and
|
|
357
|
+
* no default-tier host reaches a provisioned 1,000 MiB/s — so a bound that
|
|
358
|
+
* tightened with provisioning would page FAILED on healthy restores on
|
|
359
|
+
* exactly the hosts customers start on. The baseline-derived bound is loose
|
|
360
|
+
* on a provisioned volume, which only delays a hung-restore page; the cap
|
|
361
|
+
* keeps that delay inside the schedule interval either way. */
|
|
362
|
+
export declare function clickHouseBackupVerifyTimeoutSeconds(storageGb: number): number;
|
|
@@ -42,28 +42,66 @@ export const DEFAULT_CLICKHOUSE_INSTANCE_TYPE = "m7g.medium";
|
|
|
42
42
|
* that pulls once per launch. Upstream CH CI runs its full perf + stress
|
|
43
43
|
* matrix on the Ubuntu build; Alpine is community-tier coverage. */
|
|
44
44
|
export const CLICKHOUSE_IMAGE = "docker.io/clickhouse/clickhouse-server:26.3.17.56";
|
|
45
|
+
/** Absolute floor of the disk-critical alarm, in GiB.
|
|
46
|
+
*
|
|
47
|
+
* A pure percentage cannot express "about to break": 85 % used is 12 GiB
|
|
48
|
+
* free on an 80 GiB volume and 600 GiB free on a 4 TiB one, and only one of
|
|
49
|
+
* those is an incident. But a pure constant cannot either, because the other
|
|
50
|
+
* half of the failure scales with the data — ClickHouse will not select a
|
|
51
|
+
* merge whose source parts need more than the free space allows, and parts
|
|
52
|
+
* grow with the table. So the default floor is the LARGER of this constant
|
|
53
|
+
* and `CLICKHOUSE_DISK_FREE_CRITICAL_SHARE` of the volume: the constant
|
|
54
|
+
* guards response time on a small disk, the share guards merge headroom on a
|
|
55
|
+
* large one. See `resolveClickHouseStorage`.
|
|
56
|
+
*
|
|
57
|
+
* 20 GiB is days of ingest headroom at the observed rate — enough lead time
|
|
58
|
+
* to grow the volume, which takes an instance restart and cannot be done in
|
|
59
|
+
* minutes. */
|
|
60
|
+
export const CLICKHOUSE_DISK_FREE_CRITICAL_GIB = 20;
|
|
61
|
+
/** Share of the volume the disk-critical floor rises to on larger disks.
|
|
62
|
+
* At 5 %, a 4 TiB volume pages with 200 GiB free — roughly twice a large
|
|
63
|
+
* part, so merges still have room to complete while the operator responds.
|
|
64
|
+
* The two halves cross over at 400 GiB, the default volume size. */
|
|
65
|
+
export const CLICKHOUSE_DISK_FREE_CRITICAL_SHARE = 0.05;
|
|
45
66
|
/** EBS volume configuration.
|
|
46
67
|
*
|
|
47
|
-
*
|
|
48
|
-
* not
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
68
|
+
* `CLICKHOUSE_EBS_VOLUME_SIZE_GB` is the DEFAULT for
|
|
69
|
+
* `ClickHouseDatabaseProps.storageGb`, not a fixed size. 400 GiB is sized
|
|
70
|
+
* for a mid-size multi-tenant analytics workload — order of tens of tenant
|
|
71
|
+
* organisations each carrying full `log_events` / `application_metrics`
|
|
72
|
+
* retention windows — with enough headroom that the first resize is a
|
|
73
|
+
* deliberate capacity decision rather than an early surprise. (The
|
|
74
|
+
* reference deployment's history is the evidence: 80 GiB bound at roughly
|
|
75
|
+
* two-thirds of that scale.) gp3 storage is cheap relative to an
|
|
76
|
+
* out-of-disk ClickHouse — merges stop, inserts start failing, and the
|
|
77
|
+
* recovery is a restore rather than a resize. Workloads outside that
|
|
78
|
+
* profile set `storageGb` directly, exactly as they would size an RDS
|
|
79
|
+
* volume.
|
|
54
80
|
*
|
|
55
|
-
* Raising
|
|
56
|
-
* ext4 filesystem to match, because ModifyVolume resizes the block
|
|
57
|
-
* and nothing else. The pair moves together
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
* volume per 6 hours.
|
|
81
|
+
* Raising the size alone is not enough — `buildClickHouseUserData` must grow
|
|
82
|
+
* the ext4 filesystem to match, because ModifyVolume resizes the block
|
|
83
|
+
* device and nothing else. The pair moves together: the declared size is
|
|
84
|
+
* embedded in the user data, so a `storageGb` change versions the launch
|
|
85
|
+
* template and triggers the instance refresh whose boot runs `resize2fs`.
|
|
86
|
+
* EBS permits one modification per volume per 6 hours.
|
|
61
87
|
*
|
|
62
|
-
* IOPS and throughput
|
|
63
|
-
*
|
|
88
|
+
* IOPS and throughput default to the gp3 baseline — the figures the volume
|
|
89
|
+
* price already includes — and are the construct's `iops` / `throughputMbps`
|
|
90
|
+
* props, the way RDS exposes provisioned IOPS and throughput on gp3. They are
|
|
91
|
+
* size-independent on gp3 and the default workload is merge- and scan-bound
|
|
92
|
+
* rather than IOPS-starved, so the baseline is the right default; a caller
|
|
93
|
+
* whose scans are bandwidth-bound raises them, within the bounds and ratios
|
|
94
|
+
* `resolveClickHouseVolumePerformance` enforces. */
|
|
64
95
|
export const CLICKHOUSE_EBS_VOLUME_SIZE_GB = 400;
|
|
65
96
|
export const CLICKHOUSE_EBS_IOPS = 3000;
|
|
66
97
|
export const CLICKHOUSE_EBS_THROUGHPUT_MBPS = 125;
|
|
98
|
+
/** Volume and backup-retention bounds. Re-exported, NOT redeclared:
|
|
99
|
+
* `@fjall/generator` enforces the same numbers when it scaffolds or rewrites
|
|
100
|
+
* an `infrastructure.ts` declaration, and a second copy here is exactly the
|
|
101
|
+
* drift `backupRetentionDays` shipped for as long as it borrowed the RDS
|
|
102
|
+
* schema (generator 35 vs construct 3650). One source, two enforcement
|
|
103
|
+
* sites. */
|
|
104
|
+
export { CLICKHOUSE_MIN_STORAGE_GB, CLICKHOUSE_MAX_STORAGE_GB, CLICKHOUSE_MIN_IOPS, CLICKHOUSE_MAX_IOPS, CLICKHOUSE_MAX_IOPS_PER_GB, CLICKHOUSE_MIN_THROUGHPUT_MBPS, CLICKHOUSE_MAX_THROUGHPUT_MBPS, CLICKHOUSE_IOPS_PER_THROUGHPUT_MBPS, CLICKHOUSE_MIN_BACKUP_RETENTION_DAYS, CLICKHOUSE_MAX_BACKUP_RETENTION_DAYS, CLICKHOUSE_INSTANCE_TYPES } from "@fjall/util/clickhouse";
|
|
67
105
|
/** Host memory reserved from the ClickHouse container: kernel + ECS agent +
|
|
68
106
|
* the host-metrics timer. The ECS agent advertises MemTotal minus
|
|
69
107
|
* CLICKHOUSE_ECS_RESERVED_MEMORY_MIB, and MemTotal itself runs below the
|
|
@@ -93,6 +131,15 @@ export const CLICKHOUSE_INSTANCE_SPECS = {
|
|
|
93
131
|
"r7g.xlarge": { vcpus: 4, memoryGib: 32 },
|
|
94
132
|
"r8g.xlarge": { vcpus: 4, memoryGib: 32 }
|
|
95
133
|
};
|
|
134
|
+
/** Guarded string-keyed lookup into `CLICKHOUSE_INSTANCE_SPECS` for
|
|
135
|
+
* caller-supplied types (props carry `string`); `undefined` for unknown
|
|
136
|
+
* types, which every caller converts into its own named synth throw. The
|
|
137
|
+
* exhaustively-keyed Record above stays the compile-time contract. */
|
|
138
|
+
export function clickHouseInstanceSpec(instanceType) {
|
|
139
|
+
return Object.hasOwn(CLICKHOUSE_INSTANCE_SPECS, instanceType)
|
|
140
|
+
? CLICKHOUSE_INSTANCE_SPECS[instanceType]
|
|
141
|
+
: undefined;
|
|
142
|
+
}
|
|
96
143
|
/** Derived memory-only view of CLICKHOUSE_INSTANCE_SPECS. */
|
|
97
144
|
export const CLICKHOUSE_INSTANCE_MEMORY_GIB = Object.fromEntries(Object.entries(CLICKHOUSE_INSTANCE_SPECS).map(([type, spec]) => [
|
|
98
145
|
type,
|
|
@@ -210,6 +257,7 @@ export const CLICKHOUSE_HOST_METRICS = {
|
|
|
210
257
|
namespace: "CWAgent",
|
|
211
258
|
memoryMetric: "mem_used_percent",
|
|
212
259
|
diskMetric: "disk_used_percent",
|
|
260
|
+
diskFreeMetric: "disk_free_gib",
|
|
213
261
|
asgDimension: "AutoScalingGroupName"
|
|
214
262
|
};
|
|
215
263
|
/** Shared secret generation options (all ClickHouse users share the same policy). */
|
|
@@ -295,3 +343,66 @@ export const BACKUP_TASK_MEMORY_MIB = 256;
|
|
|
295
343
|
export const BACKUP_TASK_CPU_UNITS = 256;
|
|
296
344
|
/** Backup object expiration: 14 days (retains 14 daily snapshots). */
|
|
297
345
|
export const BACKUP_RETENTION_DAYS = 14;
|
|
346
|
+
/** Scratch database the backup task restores into to prove the backup it just
|
|
347
|
+
* wrote can be read back. Dropped on the way in and on the way out; never
|
|
348
|
+
* queried by the application. */
|
|
349
|
+
export const CLICKHOUSE_BACKUP_SCRATCH_DATABASE = "fjall_backup_verify";
|
|
350
|
+
/** Multiple of the critical free-space floor the verify restore must leave
|
|
351
|
+
* clear to proceed.
|
|
352
|
+
*
|
|
353
|
+
* A full restore writes a second copy of the database onto the volume it is
|
|
354
|
+
* verifying, so the check has to be bounded by the same quantity the disk
|
|
355
|
+
* alarm pages on — otherwise the mechanism that protects the data is also
|
|
356
|
+
* the mechanism most likely to fill the disk. Twice the floor rather than
|
|
357
|
+
* once so a verify can never itself be the thing that pages: at exactly the
|
|
358
|
+
* floor the restore would land the volume on the alarm threshold and hold it
|
|
359
|
+
* there until the scratch database is dropped. */
|
|
360
|
+
export const CLICKHOUSE_BACKUP_VERIFY_FREE_SPACE_MARGIN = 2;
|
|
361
|
+
/** Backup task log markers. Distinct full tokens, because the CloudWatch
|
|
362
|
+
* metric filters that read them match on substrings — a shared `BACKUP_`
|
|
363
|
+
* stem would make the OK filter count the failures too. */
|
|
364
|
+
export const CLICKHOUSE_BACKUP_VERIFY_OK_MARKER = "BACKUP_VERIFY_OK";
|
|
365
|
+
export const CLICKHOUSE_BACKUP_VERIFY_FAILED_MARKER = "BACKUP_VERIFY_FAILED";
|
|
366
|
+
export const CLICKHOUSE_BACKUP_VERIFY_SKIPPED_MARKER = "BACKUP_VERIFY_SKIPPED_FOR_SIZE";
|
|
367
|
+
/** Bounds on the verify restore's wall-clock ceiling, in seconds.
|
|
368
|
+
*
|
|
369
|
+
* Without a ceiling a hung RESTORE is the one backup failure with NO signal
|
|
370
|
+
* at all: BACKUP_CREATED has already been emitted so the heartbeat is
|
|
371
|
+
* satisfied, and the verify alarms count markers rather than their absence,
|
|
372
|
+
* so a task that never finishes reports nothing. `timeout` turns that
|
|
373
|
+
* silence into exit 124, which takes the same branch as a raised restore
|
|
374
|
+
* and pages.
|
|
375
|
+
*
|
|
376
|
+
* The ceiling scales with the volume (`clickHouseBackupVerifyTimeoutSeconds`
|
|
377
|
+
* below) because the largest restore the free-space gate admits scales with
|
|
378
|
+
* it: on the default 400 GiB the floor of one hour already covers it, but a
|
|
379
|
+
* multi-TiB volume can legitimately need hours, and a fixed hour would page
|
|
380
|
+
* FAILED on every healthy verify past the size where the restore outgrows
|
|
381
|
+
* it. The 20-hour cap keeps a daily-scheduled verify from overlapping its
|
|
382
|
+
* successor — two concurrent runs share the scratch database destructively,
|
|
383
|
+
* so the bound must land inside the schedule interval. */
|
|
384
|
+
export const CLICKHOUSE_BACKUP_VERIFY_MIN_TIMEOUT_SECONDS = 3600;
|
|
385
|
+
export const CLICKHOUSE_BACKUP_VERIFY_MAX_TIMEOUT_SECONDS = 72000;
|
|
386
|
+
/** Verify-restore timeout for a given data-volume size.
|
|
387
|
+
*
|
|
388
|
+
* Worst case admitted by the free-space gate is a database of roughly half
|
|
389
|
+
* the volume (it must fit beside itself). Writing that back at the gp3
|
|
390
|
+
* baseline throughput (`CLICKHOUSE_EBS_THROUGHPUT_MBPS`) bounds the healthy
|
|
391
|
+
* duration; doubled for read + merge overhead, then clamped to the
|
|
392
|
+
* floor/cap pair above. At the 400 GiB default this resolves to the
|
|
393
|
+
* one-hour floor, so the default behaviour is unchanged from the flat bound
|
|
394
|
+
* it replaced.
|
|
395
|
+
*
|
|
396
|
+
* Deliberately the BASELINE throughput, not the provisioned `throughputMbps`
|
|
397
|
+
* prop. The volume's provisioned figure is an upper bound the instance may
|
|
398
|
+
* not reach — an m7g.medium's EBS bandwidth sits under the gp3 baseline, and
|
|
399
|
+
* no default-tier host reaches a provisioned 1,000 MiB/s — so a bound that
|
|
400
|
+
* tightened with provisioning would page FAILED on healthy restores on
|
|
401
|
+
* exactly the hosts customers start on. The baseline-derived bound is loose
|
|
402
|
+
* on a provisioned volume, which only delays a hung-restore page; the cap
|
|
403
|
+
* keeps that delay inside the schedule interval either way. */
|
|
404
|
+
export function clickHouseBackupVerifyTimeoutSeconds(storageGb) {
|
|
405
|
+
const worstCaseRestoreMib = (storageGb / 2) * 1024;
|
|
406
|
+
const derived = Math.ceil((worstCaseRestoreMib / CLICKHOUSE_EBS_THROUGHPUT_MBPS) * 2);
|
|
407
|
+
return Math.min(CLICKHOUSE_BACKUP_VERIFY_MAX_TIMEOUT_SECONDS, Math.max(CLICKHOUSE_BACKUP_VERIFY_MIN_TIMEOUT_SECONDS, derived));
|
|
408
|
+
}
|
|
@@ -23,6 +23,19 @@ export type ClickHouseSchemaAdmin = z.infer<typeof ClickHouseSchemaAdminSchema>;
|
|
|
23
23
|
*/
|
|
24
24
|
export declare const ManagedPasswordNameSchema: z.ZodString;
|
|
25
25
|
export type ManagedPasswordName = z.infer<typeof ManagedPasswordNameSchema>;
|
|
26
|
+
/**
|
|
27
|
+
* One setting's constraint inside a profile's `<constraints>` block.
|
|
28
|
+
* `max`/`min` bound what a statement-level `SETTINGS` clause may set the
|
|
29
|
+
* value to; `readonly` forbids changing it at all. A statement that tries
|
|
30
|
+
* to exceed a bound fails with 452 SETTING_CONSTRAINT_VIOLATION instead of
|
|
31
|
+
* silently widening its own budget.
|
|
32
|
+
*/
|
|
33
|
+
export declare const SettingConstraintSchema: z.ZodObject<{
|
|
34
|
+
min: z.ZodOptional<z.ZodNumber>;
|
|
35
|
+
max: z.ZodOptional<z.ZodNumber>;
|
|
36
|
+
readonly: z.ZodOptional<z.ZodLiteral<true>>;
|
|
37
|
+
}, z.core.$strict>;
|
|
38
|
+
export type SettingConstraint = z.infer<typeof SettingConstraintSchema>;
|
|
26
39
|
/**
|
|
27
40
|
* Per-profile resource-cap shape. Field names mirror the ClickHouse XML
|
|
28
41
|
* element names (snake_case at the wire; camelCase here for TS ergonomics).
|
|
@@ -31,8 +44,18 @@ export type ManagedPasswordName = z.infer<typeof ManagedPasswordNameSchema>;
|
|
|
31
44
|
* `renderUsersXml` maps each present field to the snake_case XML element
|
|
32
45
|
* via `camelToSnakeCase`. Unknown fields are rejected by `.strict()` so a
|
|
33
46
|
* typo at the consumer doesn't silently ship a no-op profile.
|
|
47
|
+
*
|
|
48
|
+
* `constraints` is the one non-scalar member: it renders as the profile's
|
|
49
|
+
* `<constraints>` block (keys camelCase here, snake_case at the wire, same
|
|
50
|
+
* as the scalar settings), making the named ceilings unraisable from a
|
|
51
|
+
* statement-level `SETTINGS` clause.
|
|
34
52
|
*/
|
|
35
53
|
export declare const ProfileSpecSchema: z.ZodObject<{
|
|
54
|
+
constraints: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
55
|
+
min: z.ZodOptional<z.ZodNumber>;
|
|
56
|
+
max: z.ZodOptional<z.ZodNumber>;
|
|
57
|
+
readonly: z.ZodOptional<z.ZodLiteral<true>>;
|
|
58
|
+
}, z.core.$strict>>>;
|
|
36
59
|
maxThreads: z.ZodOptional<z.ZodNumber>;
|
|
37
60
|
maxInsertThreads: z.ZodOptional<z.ZodNumber>;
|
|
38
61
|
maxConcurrentQueriesForUser: z.ZodOptional<z.ZodNumber>;
|
|
@@ -62,6 +85,15 @@ export declare const ProfileSpecSchema: z.ZodObject<{
|
|
|
62
85
|
materializeTtlAfterModify: z.ZodOptional<z.ZodBoolean>;
|
|
63
86
|
}, z.core.$strict>;
|
|
64
87
|
export type ProfileSpec = z.infer<typeof ProfileSpecSchema>;
|
|
88
|
+
/**
|
|
89
|
+
* Deep-frozen `ProfileSpec` — matches the `Object.freeze` depth of the
|
|
90
|
+
* derived defaults so the type system forbids what the runtime freeze
|
|
91
|
+
* throws on. `Readonly<ProfileSpec>` alone is shallow: it left the nested
|
|
92
|
+
* constraint objects typed mutable while frozen at runtime.
|
|
93
|
+
*/
|
|
94
|
+
export type FrozenProfileSpec = Readonly<Omit<ProfileSpec, "constraints">> & {
|
|
95
|
+
readonly constraints?: Readonly<Record<string, Readonly<SettingConstraint>>>;
|
|
96
|
+
};
|
|
65
97
|
/**
|
|
66
98
|
* Re-exported for the construct's Stage 1 validation. Profile keys MUST
|
|
67
99
|
* match the same lowercase snake_case shape as user names — they emit as
|
|
@@ -88,4 +120,4 @@ export declare const PROFILE_NAME_PATTERN: RegExp;
|
|
|
88
120
|
* consumer cannot mutate the defaults in place. Pass your own `profiles:`
|
|
89
121
|
* map to the construct to extend / override; never mutate this constant.
|
|
90
122
|
*/
|
|
91
|
-
export declare const ClickHouseDefaultProfiles: Readonly<Record<string,
|
|
123
|
+
export declare const ClickHouseDefaultProfiles: Readonly<Record<string, FrozenProfileSpec>>;
|
|
@@ -9,6 +9,12 @@ import { deriveClickHouseDefaultProfiles } from "./clickhouseTuning.js";
|
|
|
9
9
|
* surfaces as a parse failure in the runner). Keeps the XML element names
|
|
10
10
|
* emit-safe (no escaping needed). */
|
|
11
11
|
const NAME_PATTERN = /^[a-z][a-z0-9_]*$/;
|
|
12
|
+
/** Constraint-record keys — camelCase (or snake_case) setting names with a
|
|
13
|
+
* leading lowercase letter. Record keys are the one profile surface
|
|
14
|
+
* `.strict()` cannot police, and they emit as XML element names via
|
|
15
|
+
* `camelToSnakeCase`, so the emit-safety NAME_PATTERN guarantees for
|
|
16
|
+
* user/profile names must come from the key schema itself here. */
|
|
17
|
+
const CONSTRAINT_KEY_PATTERN = /^[a-z][a-zA-Z0-9_]*$/;
|
|
12
18
|
/**
|
|
13
19
|
* Schema-admin user — the framework's bootstrap-privileged identity that
|
|
14
20
|
* customer SQL runs as. Owns DDL, schema migrations, GRANTs. Rendered into
|
|
@@ -41,6 +47,21 @@ export const ManagedPasswordNameSchema = z
|
|
|
41
47
|
.min(1)
|
|
42
48
|
.max(63)
|
|
43
49
|
.regex(NAME_PATTERN, "Must be lowercase snake_case");
|
|
50
|
+
/**
|
|
51
|
+
* One setting's constraint inside a profile's `<constraints>` block.
|
|
52
|
+
* `max`/`min` bound what a statement-level `SETTINGS` clause may set the
|
|
53
|
+
* value to; `readonly` forbids changing it at all. A statement that tries
|
|
54
|
+
* to exceed a bound fails with 452 SETTING_CONSTRAINT_VIOLATION instead of
|
|
55
|
+
* silently widening its own budget.
|
|
56
|
+
*/
|
|
57
|
+
export const SettingConstraintSchema = z
|
|
58
|
+
.object({
|
|
59
|
+
min: z.number().int().nonnegative().optional(),
|
|
60
|
+
max: z.number().int().nonnegative().optional(),
|
|
61
|
+
readonly: z.literal(true).optional()
|
|
62
|
+
})
|
|
63
|
+
.strict()
|
|
64
|
+
.refine((c) => c.min !== undefined || c.max !== undefined || c.readonly === true, "A setting constraint must declare min, max or readonly");
|
|
44
65
|
/**
|
|
45
66
|
* Per-profile resource-cap shape. Field names mirror the ClickHouse XML
|
|
46
67
|
* element names (snake_case at the wire; camelCase here for TS ergonomics).
|
|
@@ -49,9 +70,19 @@ export const ManagedPasswordNameSchema = z
|
|
|
49
70
|
* `renderUsersXml` maps each present field to the snake_case XML element
|
|
50
71
|
* via `camelToSnakeCase`. Unknown fields are rejected by `.strict()` so a
|
|
51
72
|
* typo at the consumer doesn't silently ship a no-op profile.
|
|
73
|
+
*
|
|
74
|
+
* `constraints` is the one non-scalar member: it renders as the profile's
|
|
75
|
+
* `<constraints>` block (keys camelCase here, snake_case at the wire, same
|
|
76
|
+
* as the scalar settings), making the named ceilings unraisable from a
|
|
77
|
+
* statement-level `SETTINGS` clause.
|
|
52
78
|
*/
|
|
53
79
|
export const ProfileSpecSchema = z
|
|
54
80
|
.object({
|
|
81
|
+
constraints: z
|
|
82
|
+
.record(z
|
|
83
|
+
.string()
|
|
84
|
+
.regex(CONSTRAINT_KEY_PATTERN, "Constraint keys must be camelCase (or snake_case) setting names"), SettingConstraintSchema)
|
|
85
|
+
.optional(),
|
|
55
86
|
maxThreads: z.number().int().positive().optional(),
|
|
56
87
|
maxInsertThreads: z.number().int().positive().optional(),
|
|
57
88
|
maxConcurrentQueriesForUser: z.number().int().positive().optional(),
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Default critical free-space floor for a volume of `storageGb`.
|
|
3
|
+
*
|
|
4
|
+
* The larger of the absolute floor and a share of the volume — neither alone
|
|
5
|
+
* is right at both ends of the size range. See
|
|
6
|
+
* `CLICKHOUSE_DISK_FREE_CRITICAL_GIB` for the reasoning; rounded to whole GiB
|
|
7
|
+
* so the alarm description reads as a number an operator would say out loud.
|
|
8
|
+
*/
|
|
9
|
+
export declare function defaultDiskFreeCriticalGib(storageGb: number): number;
|
|
10
|
+
export interface ClickHouseStorageResolution {
|
|
11
|
+
/** EBS volume size in GiB, defaulted and validated. */
|
|
12
|
+
storageGb: number;
|
|
13
|
+
/** Critical free-space floor in GiB, defaulted and validated. */
|
|
14
|
+
diskFreeCriticalGib: number;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Resolve and validate the two values that together decide when a ClickHouse
|
|
18
|
+
* volume is considered nearly full.
|
|
19
|
+
*
|
|
20
|
+
* They are validated as a PAIR because neither is checkable alone: a 100 GiB
|
|
21
|
+
* volume is fine until someone sets a 60 GiB free-space floor, and a 60 GiB
|
|
22
|
+
* floor is fine until someone shrinks the volume under it. Split across two
|
|
23
|
+
* validators, each would pass on its own and the deployed stack would carry a
|
|
24
|
+
* critical alarm that can never leave ALARM.
|
|
25
|
+
*
|
|
26
|
+
* Throws rather than annotating: both are numbers the caller typed, and a
|
|
27
|
+
* warning on a synth that still emits a permanently-breaching pager is worse
|
|
28
|
+
* than a failed synth with the arithmetic in the message.
|
|
29
|
+
*/
|
|
30
|
+
export declare function resolveClickHouseStorage(storageGb: number | undefined, diskFreeCriticalGib: number | undefined): ClickHouseStorageResolution;
|
|
31
|
+
export interface ClickHouseVolumePerformanceInput {
|
|
32
|
+
/** Validated volume size from `resolveClickHouseStorage` — the IOPS ceiling
|
|
33
|
+
* is a function of it. */
|
|
34
|
+
storageGb: number;
|
|
35
|
+
iops: number | undefined;
|
|
36
|
+
throughputMbps: number | undefined;
|
|
37
|
+
}
|
|
38
|
+
export interface ClickHouseVolumePerformanceResolution {
|
|
39
|
+
/** Provisioned IOPS, defaulted and validated. */
|
|
40
|
+
iops: number;
|
|
41
|
+
/** Provisioned throughput in MiB/s, defaulted and validated. */
|
|
42
|
+
throughputMbps: number;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Resolve and validate the gp3 performance pair for the data volume.
|
|
46
|
+
*
|
|
47
|
+
* Same shape as `resolveClickHouseStorage`: each value has a band of its own,
|
|
48
|
+
* but EBS also constrains them against each other and against the size —
|
|
49
|
+
* throughput needs four IOPS per MiB/s, and IOPS needs a GiB per 500 — so a
|
|
50
|
+
* value legal on its own is rejected by CreateVolume mid-deploy once its
|
|
51
|
+
* partner is in view. Checking the pair at synth turns that rollback into a
|
|
52
|
+
* message that names the figure the other half needs.
|
|
53
|
+
*
|
|
54
|
+
* Both checks are the ones aws-cdk-lib's `Volume` applies too; this resolver
|
|
55
|
+
* exists so the failure names `ClickHouseDatabase` props and their cure
|
|
56
|
+
* rather than a CDK-internal field.
|
|
57
|
+
*/
|
|
58
|
+
export declare function resolveClickHouseVolumePerformance(input: ClickHouseVolumePerformanceInput): ClickHouseVolumePerformanceResolution;
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { CLICKHOUSE_DISK_FREE_CRITICAL_GIB, CLICKHOUSE_DISK_FREE_CRITICAL_SHARE, CLICKHOUSE_EBS_IOPS, CLICKHOUSE_EBS_THROUGHPUT_MBPS, CLICKHOUSE_EBS_VOLUME_SIZE_GB, CLICKHOUSE_IOPS_PER_THROUGHPUT_MBPS, CLICKHOUSE_MAX_IOPS, CLICKHOUSE_MAX_IOPS_PER_GB, CLICKHOUSE_MAX_STORAGE_GB, CLICKHOUSE_MAX_THROUGHPUT_MBPS, CLICKHOUSE_MIN_IOPS, CLICKHOUSE_MIN_STORAGE_GB, CLICKHOUSE_MIN_THROUGHPUT_MBPS } from "./clickhouseConstants.js";
|
|
2
|
+
/**
|
|
3
|
+
* Largest share of the volume the critical free-space floor may claim. A
|
|
4
|
+
* floor at half the disk would page from the day the cluster passed 50 %
|
|
5
|
+
* used, which is a capacity-planning fact, not an incident; at a fifth, the
|
|
6
|
+
* alarm means what its description says — the disk is nearly gone.
|
|
7
|
+
*/
|
|
8
|
+
const MAX_FLOOR_SHARE_OF_VOLUME = 0.2;
|
|
9
|
+
/**
|
|
10
|
+
* Default critical free-space floor for a volume of `storageGb`.
|
|
11
|
+
*
|
|
12
|
+
* The larger of the absolute floor and a share of the volume — neither alone
|
|
13
|
+
* is right at both ends of the size range. See
|
|
14
|
+
* `CLICKHOUSE_DISK_FREE_CRITICAL_GIB` for the reasoning; rounded to whole GiB
|
|
15
|
+
* so the alarm description reads as a number an operator would say out loud.
|
|
16
|
+
*/
|
|
17
|
+
export function defaultDiskFreeCriticalGib(storageGb) {
|
|
18
|
+
return Math.max(CLICKHOUSE_DISK_FREE_CRITICAL_GIB, Math.round(storageGb * CLICKHOUSE_DISK_FREE_CRITICAL_SHARE));
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Resolve and validate the two values that together decide when a ClickHouse
|
|
22
|
+
* volume is considered nearly full.
|
|
23
|
+
*
|
|
24
|
+
* They are validated as a PAIR because neither is checkable alone: a 100 GiB
|
|
25
|
+
* volume is fine until someone sets a 60 GiB free-space floor, and a 60 GiB
|
|
26
|
+
* floor is fine until someone shrinks the volume under it. Split across two
|
|
27
|
+
* validators, each would pass on its own and the deployed stack would carry a
|
|
28
|
+
* critical alarm that can never leave ALARM.
|
|
29
|
+
*
|
|
30
|
+
* Throws rather than annotating: both are numbers the caller typed, and a
|
|
31
|
+
* warning on a synth that still emits a permanently-breaching pager is worse
|
|
32
|
+
* than a failed synth with the arithmetic in the message.
|
|
33
|
+
*/
|
|
34
|
+
export function resolveClickHouseStorage(storageGb, diskFreeCriticalGib) {
|
|
35
|
+
const size = storageGb ?? CLICKHOUSE_EBS_VOLUME_SIZE_GB;
|
|
36
|
+
if (!Number.isInteger(size) ||
|
|
37
|
+
size < CLICKHOUSE_MIN_STORAGE_GB ||
|
|
38
|
+
size > CLICKHOUSE_MAX_STORAGE_GB) {
|
|
39
|
+
throw new Error(`ClickHouseDatabase: storageGb must be an integer between ${CLICKHOUSE_MIN_STORAGE_GB} and ${CLICKHOUSE_MAX_STORAGE_GB} GiB; got ${size}.`);
|
|
40
|
+
}
|
|
41
|
+
const floor = diskFreeCriticalGib ?? defaultDiskFreeCriticalGib(size);
|
|
42
|
+
if (!Number.isFinite(floor) || floor <= 0) {
|
|
43
|
+
throw new Error(`ClickHouseDatabase: alarms.diskFreeCriticalGib must be a positive number of GiB; got ${floor}.`);
|
|
44
|
+
}
|
|
45
|
+
const ceiling = size * MAX_FLOOR_SHARE_OF_VOLUME;
|
|
46
|
+
if (floor > ceiling) {
|
|
47
|
+
throw new Error(`ClickHouseDatabase: alarms.diskFreeCriticalGib (${floor} GiB) exceeds a fifth of storageGb (${size} GiB → ${ceiling} GiB). A floor that large keeps the disk-critical alarm in ALARM on a healthy cluster; raise storageGb or lower the floor.`);
|
|
48
|
+
}
|
|
49
|
+
return { storageGb: size, diskFreeCriticalGib: floor };
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Resolve and validate the gp3 performance pair for the data volume.
|
|
53
|
+
*
|
|
54
|
+
* Same shape as `resolveClickHouseStorage`: each value has a band of its own,
|
|
55
|
+
* but EBS also constrains them against each other and against the size —
|
|
56
|
+
* throughput needs four IOPS per MiB/s, and IOPS needs a GiB per 500 — so a
|
|
57
|
+
* value legal on its own is rejected by CreateVolume mid-deploy once its
|
|
58
|
+
* partner is in view. Checking the pair at synth turns that rollback into a
|
|
59
|
+
* message that names the figure the other half needs.
|
|
60
|
+
*
|
|
61
|
+
* Both checks are the ones aws-cdk-lib's `Volume` applies too; this resolver
|
|
62
|
+
* exists so the failure names `ClickHouseDatabase` props and their cure
|
|
63
|
+
* rather than a CDK-internal field.
|
|
64
|
+
*/
|
|
65
|
+
export function resolveClickHouseVolumePerformance(input) {
|
|
66
|
+
const iops = input.iops ?? CLICKHOUSE_EBS_IOPS;
|
|
67
|
+
if (!Number.isInteger(iops) ||
|
|
68
|
+
iops < CLICKHOUSE_MIN_IOPS ||
|
|
69
|
+
iops > CLICKHOUSE_MAX_IOPS) {
|
|
70
|
+
throw new Error(`ClickHouseDatabase: iops must be an integer between ${CLICKHOUSE_MIN_IOPS} and ${CLICKHOUSE_MAX_IOPS}; got ${iops}.`);
|
|
71
|
+
}
|
|
72
|
+
const iopsCeilingForSize = input.storageGb * CLICKHOUSE_MAX_IOPS_PER_GB;
|
|
73
|
+
if (iops > iopsCeilingForSize) {
|
|
74
|
+
throw new Error(`ClickHouseDatabase: iops (${iops}) exceeds ${CLICKHOUSE_MAX_IOPS_PER_GB} per GiB of storageGb (${input.storageGb} GiB → ${iopsCeilingForSize}). EBS rejects the volume; raise storageGb to at least ${Math.ceil(iops / CLICKHOUSE_MAX_IOPS_PER_GB)} GiB or lower iops.`);
|
|
75
|
+
}
|
|
76
|
+
const throughputMbps = input.throughputMbps ?? CLICKHOUSE_EBS_THROUGHPUT_MBPS;
|
|
77
|
+
if (!Number.isInteger(throughputMbps) ||
|
|
78
|
+
throughputMbps < CLICKHOUSE_MIN_THROUGHPUT_MBPS ||
|
|
79
|
+
throughputMbps > CLICKHOUSE_MAX_THROUGHPUT_MBPS) {
|
|
80
|
+
throw new Error(`ClickHouseDatabase: throughputMbps must be an integer between ${CLICKHOUSE_MIN_THROUGHPUT_MBPS} and ${CLICKHOUSE_MAX_THROUGHPUT_MBPS} MiB/s; got ${throughputMbps}.`);
|
|
81
|
+
}
|
|
82
|
+
const iopsNeeded = throughputMbps * CLICKHOUSE_IOPS_PER_THROUGHPUT_MBPS;
|
|
83
|
+
if (iops < iopsNeeded) {
|
|
84
|
+
throw new Error(`ClickHouseDatabase: throughputMbps (${throughputMbps} MiB/s) needs ${CLICKHOUSE_IOPS_PER_THROUGHPUT_MBPS} IOPS per MiB/s, so at least ${iopsNeeded} iops; got ${iops}. EBS rejects the volume; raise iops to ${iopsNeeded} or lower throughputMbps to ${Math.floor(iops / CLICKHOUSE_IOPS_PER_THROUGHPUT_MBPS)}.`);
|
|
85
|
+
}
|
|
86
|
+
return { iops, throughputMbps };
|
|
87
|
+
}
|