@fjall/components-infrastructure 10.1.2 → 11.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/patterns/aws/clickhouseDatabase.d.ts +13 -7
- package/dist/lib/patterns/aws/clickhouseDatabase.js +21 -16
- package/dist/lib/patterns/aws/computeEcs.js +7 -1
- package/dist/lib/resources/aws/compute/ecs.js +1 -1
- package/dist/lib/resources/aws/compute/ecsConstants.d.ts +36 -0
- package/dist/lib/resources/aws/compute/ecsConstants.js +113 -0
- package/dist/lib/resources/aws/compute/ecsTaskDefinition.js +8 -2
- package/dist/lib/resources/aws/compute/ecsValidation.d.ts +69 -1
- package/dist/lib/resources/aws/compute/ecsValidation.js +113 -2
- package/dist/lib/resources/aws/database/clickhouseConstants.d.ts +24 -9
- package/dist/lib/resources/aws/database/clickhouseConstants.js +32 -23
- package/dist/lib/resources/aws/database/clickhouseSchemas.d.ts +20 -0
- package/dist/lib/resources/aws/database/clickhouseSchemas.js +20 -77
- package/dist/lib/resources/aws/database/clickhouseTuning.d.ts +64 -0
- package/dist/lib/resources/aws/database/clickhouseTuning.js +182 -0
- package/dist/lib/resources/aws/database/clickhouseUserData.d.ts +7 -0
- package/dist/lib/resources/aws/database/clickhouseUserData.js +87 -52
- package/dist/lib/resources/aws/database/clickhouseXmlRenderer.js +10 -0
- package/package.json +3 -3
|
@@ -10,19 +10,17 @@ export const CLICKHOUSE_DATABASE_NAME = "analytics";
|
|
|
10
10
|
* of 2 vCPU) for the rest of the hour. Graviton3 is also ~25% faster on
|
|
11
11
|
* columnar scans than Graviton2.
|
|
12
12
|
*
|
|
13
|
-
* Settings are tuned for 1 vCPU sustained (max_threads=1 across all profiles,
|
|
14
|
-
* max_concurrent_queries=8) — see clickhouseUserData.ts. Bumping to a larger
|
|
15
|
-
* instance MUST be paired with raising those thread/concurrency caps.
|
|
16
|
-
*
|
|
17
13
|
* Next size up (2026-07-31 right-sizing, designs/2026-07-31-clickhouse-
|
|
18
14
|
* factory-fit-for-purpose.md): `r8g.medium` (1 vCPU Graviton4, 8 GiB,
|
|
19
15
|
* ~+30%/core over Graviton3) — CH's own doctrine floors production at
|
|
20
16
|
* 8 GiB, and the memory-bound single-tenant profile wants RAM before
|
|
21
17
|
* cores. `m7g.large` (2 vCPU, 8 GiB) only when CPU-bound. Set via
|
|
22
18
|
* `clickhouseInstanceType` CDK context or the `instanceType` prop —
|
|
23
|
-
* the container memory limit
|
|
24
|
-
*
|
|
25
|
-
*
|
|
19
|
+
* the container memory limit, server pools/caches, and profile resource
|
|
20
|
+
* caps ALL derive from the type automatically (`clickHouseTaskMemoryMiB()`,
|
|
21
|
+
* `deriveClickHouseServerTuning()`, `deriveClickHouseDefaultProfiles()`).
|
|
22
|
+
* A size change needs no manual re-tune — but it DOES need an instance
|
|
23
|
+
* refresh + server restart to take effect (config is baked into user-data). */
|
|
26
24
|
export const DEFAULT_CLICKHOUSE_INSTANCE_TYPE = "m7g.medium";
|
|
27
25
|
/** ClickHouse container image. Explicit `docker.io/` prefix is required so
|
|
28
26
|
* string-form consumers in `ecsImages.ts#getContainerImage()` route through
|
|
@@ -62,23 +60,26 @@ export const CLICKHOUSE_HOST_RESERVE_MIB = 1024;
|
|
|
62
60
|
* derived from nominal memory, so the host reserve is what absorbs this
|
|
63
61
|
* withholding at task placement. */
|
|
64
62
|
export const CLICKHOUSE_ECS_RESERVED_MEMORY_MIB = 256;
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
"
|
|
69
|
-
"
|
|
70
|
-
"
|
|
71
|
-
"
|
|
72
|
-
"
|
|
73
|
-
"
|
|
74
|
-
"
|
|
75
|
-
"
|
|
76
|
-
"
|
|
77
|
-
"
|
|
78
|
-
"
|
|
79
|
-
"r7g.xlarge": 32,
|
|
80
|
-
"r8g.xlarge": 32
|
|
63
|
+
export const CLICKHOUSE_INSTANCE_SPECS = {
|
|
64
|
+
"t4g.medium": { vcpus: 2, memoryGib: 4 },
|
|
65
|
+
"m7g.medium": { vcpus: 1, memoryGib: 4 },
|
|
66
|
+
"m8g.medium": { vcpus: 1, memoryGib: 4 },
|
|
67
|
+
"r7g.medium": { vcpus: 1, memoryGib: 8 },
|
|
68
|
+
"r8g.medium": { vcpus: 1, memoryGib: 8 },
|
|
69
|
+
"m7g.large": { vcpus: 2, memoryGib: 8 },
|
|
70
|
+
"m8g.large": { vcpus: 2, memoryGib: 8 },
|
|
71
|
+
"r7g.large": { vcpus: 2, memoryGib: 16 },
|
|
72
|
+
"r8g.large": { vcpus: 2, memoryGib: 16 },
|
|
73
|
+
"m7g.xlarge": { vcpus: 4, memoryGib: 16 },
|
|
74
|
+
"m8g.xlarge": { vcpus: 4, memoryGib: 16 },
|
|
75
|
+
"r7g.xlarge": { vcpus: 4, memoryGib: 32 },
|
|
76
|
+
"r8g.xlarge": { vcpus: 4, memoryGib: 32 }
|
|
81
77
|
};
|
|
78
|
+
/** Derived memory-only view of CLICKHOUSE_INSTANCE_SPECS. */
|
|
79
|
+
export const CLICKHOUSE_INSTANCE_MEMORY_GIB = Object.fromEntries(Object.entries(CLICKHOUSE_INSTANCE_SPECS).map(([type, spec]) => [
|
|
80
|
+
type,
|
|
81
|
+
spec.memoryGib
|
|
82
|
+
]));
|
|
82
83
|
/** ECS container memory for the ClickHouse server task, derived from the
|
|
83
84
|
* instance type so a size change cannot leave the hand-set container cap
|
|
84
85
|
* behind (the pre-2026-07 shape: a hardcoded 3072 that silently starved —
|
|
@@ -241,6 +242,14 @@ export const CLICKHOUSE_CLOUDMAP_SERVICE_NAME = "clickhouse";
|
|
|
241
242
|
* live-reload path into a silent no-op without any compile/test signal,
|
|
242
243
|
* so this lives as a single source of truth. */
|
|
243
244
|
export const CLICKHOUSE_SERVER_CONTAINER_NAME = "clickhouse";
|
|
245
|
+
/** ECS service name for the ClickHouse server. Coupled cross-repo value:
|
|
246
|
+
* the webapp log collector excludes this service from customer log
|
|
247
|
+
* collection by exact name (`CLICKHOUSE_SERVER_SERVICE_NAME` at
|
|
248
|
+
* `webapp/app/.server/services/monitoring/logGroupResolution.ts`) —
|
|
249
|
+
* deliberately duplicated rather than shared, so a webapp deploy is never
|
|
250
|
+
* release-coupled to a constructs mint. Renaming this service breaks that
|
|
251
|
+
* exclusion silently; update both sites together. */
|
|
252
|
+
export const CLICKHOUSE_SERVICE_NAME = "ClickHouseService";
|
|
244
253
|
/** Materialised views that benefit from periodic OPTIMIZE to reduce part count at read time.
|
|
245
254
|
* These are not ReplacingMergeTree (no dedup needed) but un-merged parts force
|
|
246
255
|
* read-time aggregation which degrades query performance.
|
|
@@ -68,4 +68,24 @@ export type ProfileSpec = z.infer<typeof ProfileSpecSchema>;
|
|
|
68
68
|
* XML element names with no escaping.
|
|
69
69
|
*/
|
|
70
70
|
export declare const PROFILE_NAME_PATTERN: RegExp;
|
|
71
|
+
/**
|
|
72
|
+
* Four workload-class default profiles at the DEFAULT instance type's scale.
|
|
73
|
+
* Since the 2026-08-12 instance-scaled derivation the canonical source is
|
|
74
|
+
* `deriveClickHouseDefaultProfiles(instanceType)` in `clickhouseTuning.ts` —
|
|
75
|
+
* the construct derives its default `profiles:` from the RESOLVED instance
|
|
76
|
+
* type, so this constant serves only consumers with no instance in hand
|
|
77
|
+
* (parity tests, docs). At `DEFAULT_CLICKHOUSE_INSTANCE_TYPE` the derivation
|
|
78
|
+
* reproduces the historical hand-tuned values byte-for-byte (pinned by
|
|
79
|
+
* clickhouseTuning.test.ts).
|
|
80
|
+
*
|
|
81
|
+
* `high_throughput_ingest.maxConcurrentQueriesForUser` (6) mirrors the
|
|
82
|
+
* webapp's client-side FIFO gate
|
|
83
|
+
* (`webapp/app/.server/lib/clickhouse/appQuerySlots.ts`) — a users.xml
|
|
84
|
+
* parity test enforces the pair, and the derivation deliberately never
|
|
85
|
+
* scales it.
|
|
86
|
+
*
|
|
87
|
+
* Frozen via `Object.freeze` on the outer record AND each sub-object so a
|
|
88
|
+
* consumer cannot mutate the defaults in place. Pass your own `profiles:`
|
|
89
|
+
* map to the construct to extend / override; never mutate this constant.
|
|
90
|
+
*/
|
|
71
91
|
export declare const ClickHouseDefaultProfiles: Readonly<Record<string, Readonly<ProfileSpec>>>;
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
+
import { DEFAULT_CLICKHOUSE_INSTANCE_TYPE } from "./clickhouseConstants.js";
|
|
3
|
+
import { deriveClickHouseDefaultProfiles } from "./clickhouseTuning.js";
|
|
2
4
|
/** ClickHouse user/profile name pattern — lowercase snake_case, leading
|
|
3
5
|
* letter only. Shared by `ClickHouseSchemaAdminSchema.name`, the per-profile
|
|
4
6
|
* key regex applied inside `ClickHouseDatabase`'s construct-time validation
|
|
@@ -86,82 +88,23 @@ export const ProfileSpecSchema = z
|
|
|
86
88
|
*/
|
|
87
89
|
export const PROFILE_NAME_PATTERN = NAME_PATTERN;
|
|
88
90
|
/**
|
|
89
|
-
* Four workload-class default profiles
|
|
90
|
-
*
|
|
91
|
-
*
|
|
92
|
-
*
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
* the
|
|
96
|
-
*
|
|
97
|
-
* queue past this cap — `queue_max_wait_ms` empirically does not apply).
|
|
98
|
-
* Consumers pair it with a client-side FIFO gate
|
|
99
|
-
* (`webapp/app/.server/lib/clickhouse/appQuerySlots.ts`) whose limit
|
|
100
|
-
* MUST mirror this value — a users.xml parity test enforces the pair.
|
|
91
|
+
* Four workload-class default profiles at the DEFAULT instance type's scale.
|
|
92
|
+
* Since the 2026-08-12 instance-scaled derivation the canonical source is
|
|
93
|
+
* `deriveClickHouseDefaultProfiles(instanceType)` in `clickhouseTuning.ts` —
|
|
94
|
+
* the construct derives its default `profiles:` from the RESOLVED instance
|
|
95
|
+
* type, so this constant serves only consumers with no instance in hand
|
|
96
|
+
* (parity tests, docs). At `DEFAULT_CLICKHOUSE_INSTANCE_TYPE` the derivation
|
|
97
|
+
* reproduces the historical hand-tuned values byte-for-byte (pinned by
|
|
98
|
+
* clickhouseTuning.test.ts).
|
|
101
99
|
*
|
|
102
|
-
*
|
|
103
|
-
*
|
|
104
|
-
* `
|
|
105
|
-
*
|
|
100
|
+
* `high_throughput_ingest.maxConcurrentQueriesForUser` (6) mirrors the
|
|
101
|
+
* webapp's client-side FIFO gate
|
|
102
|
+
* (`webapp/app/.server/lib/clickhouse/appQuerySlots.ts`) — a users.xml
|
|
103
|
+
* parity test enforces the pair, and the derivation deliberately never
|
|
104
|
+
* scales it.
|
|
105
|
+
*
|
|
106
|
+
* Frozen via `Object.freeze` on the outer record AND each sub-object so a
|
|
107
|
+
* consumer cannot mutate the defaults in place. Pass your own `profiles:`
|
|
108
|
+
* map to the construct to extend / override; never mutate this constant.
|
|
106
109
|
*/
|
|
107
|
-
const
|
|
108
|
-
high_throughput_ingest: {
|
|
109
|
-
maxThreads: 1,
|
|
110
|
-
maxInsertThreads: 1,
|
|
111
|
-
maxConcurrentQueriesForUser: 6,
|
|
112
|
-
logQueriesMinQueryDurationMs: 100,
|
|
113
|
-
optimizeMoveToPrewhere: true,
|
|
114
|
-
useQueryConditionCache: true,
|
|
115
|
-
useSkipIndexesIfFinal: true,
|
|
116
|
-
applyRowPolicyAfterFinal: true,
|
|
117
|
-
applyPrewhereAfterFinal: true,
|
|
118
|
-
doNotMergeAcrossPartitionsSelectFinal: true,
|
|
119
|
-
asyncInsert: true,
|
|
120
|
-
waitForAsyncInsert: true,
|
|
121
|
-
asyncInsertMaxDataSize: 10000000,
|
|
122
|
-
asyncInsertBusyTimeoutMinMs: 50,
|
|
123
|
-
asyncInsertBusyTimeoutMaxMs: 2000,
|
|
124
|
-
asyncInsertUseAdaptiveBusyTimeout: true,
|
|
125
|
-
asyncInsertDeduplicate: true,
|
|
126
|
-
inputFormatParallelParsing: false,
|
|
127
|
-
outputFormatParallelFormatting: false,
|
|
128
|
-
queryPlanOptimizeLazyMaterialization: true,
|
|
129
|
-
maxMemoryUsage: 1610612736,
|
|
130
|
-
maxMemoryUsageForUser: 2147483648,
|
|
131
|
-
maxBytesBeforeExternalSort: 536870912,
|
|
132
|
-
maxBytesBeforeExternalGroupBy: 536870912,
|
|
133
|
-
maxExecutionTime: 30,
|
|
134
|
-
maxRowsToRead: 10000000
|
|
135
|
-
},
|
|
136
|
-
audit_append: {
|
|
137
|
-
maxThreads: 1,
|
|
138
|
-
maxInsertThreads: 1,
|
|
139
|
-
maxConcurrentQueriesForUser: 2,
|
|
140
|
-
maxMemoryUsage: 500000000,
|
|
141
|
-
maxExecutionTime: 10,
|
|
142
|
-
asyncInsert: true,
|
|
143
|
-
waitForAsyncInsert: true
|
|
144
|
-
},
|
|
145
|
-
read_only: {
|
|
146
|
-
maxThreads: 1,
|
|
147
|
-
maxConcurrentQueriesForUser: 1,
|
|
148
|
-
maxMemoryUsage: 1000000000,
|
|
149
|
-
maxExecutionTime: 3600
|
|
150
|
-
},
|
|
151
|
-
ddl_admin: {
|
|
152
|
-
maxThreads: 1,
|
|
153
|
-
maxConcurrentQueriesForUser: 1,
|
|
154
|
-
maxMemoryUsage: 1000000000,
|
|
155
|
-
maxExecutionTime: 1800
|
|
156
|
-
}
|
|
157
|
-
};
|
|
158
|
-
export const ClickHouseDefaultProfiles = Object.freeze({
|
|
159
|
-
high_throughput_ingest: Object.freeze({
|
|
160
|
-
..._ClickHouseDefaultProfilesRaw.high_throughput_ingest
|
|
161
|
-
}),
|
|
162
|
-
audit_append: Object.freeze({
|
|
163
|
-
..._ClickHouseDefaultProfilesRaw.audit_append
|
|
164
|
-
}),
|
|
165
|
-
read_only: Object.freeze({ ..._ClickHouseDefaultProfilesRaw.read_only }),
|
|
166
|
-
ddl_admin: Object.freeze({ ..._ClickHouseDefaultProfilesRaw.ddl_admin })
|
|
167
|
-
});
|
|
110
|
+
export const ClickHouseDefaultProfiles = deriveClickHouseDefaultProfiles(DEFAULT_CLICKHOUSE_INSTANCE_TYPE);
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import type { ProfileSpec } from "./clickhouseSchemas.js";
|
|
2
|
+
/**
|
|
3
|
+
* Server-level tuning values derived from the instance type. Consumed by
|
|
4
|
+
* `generateServerConfigXml` — every value here lands in the rendered
|
|
5
|
+
* `config.d/fjall.xml`, so a change means a launch-template refresh +
|
|
6
|
+
* instance refresh + server restart to take effect (the schedule-pool
|
|
7
|
+
* per-type cap in particular is computed once at pool creation).
|
|
8
|
+
*/
|
|
9
|
+
export interface ClickHouseServerTuning {
|
|
10
|
+
maxConcurrentQueries: number;
|
|
11
|
+
backgroundPoolSize: number;
|
|
12
|
+
backgroundSchedulePoolSize: number;
|
|
13
|
+
backgroundMovePoolSize: number;
|
|
14
|
+
backgroundFetchesPoolSize: number;
|
|
15
|
+
backgroundMessageBrokerSchedulePoolSize: number;
|
|
16
|
+
mergesMutationsConcurrencyRatio: number;
|
|
17
|
+
freeEntriesToExecuteMutation: number;
|
|
18
|
+
freeEntriesToLowerMaxSizeOfMerge: number;
|
|
19
|
+
freeEntriesToExecuteOptimizeEntirePartition: number;
|
|
20
|
+
markCacheBytes: number;
|
|
21
|
+
indexMarkCacheBytes: number;
|
|
22
|
+
queryCacheBytes: number;
|
|
23
|
+
}
|
|
24
|
+
/** Ratio ClickHouse >= 25.9 applies to the schedule pool to derive the
|
|
25
|
+
* per-type concurrent-task cap (`background_schedule_pool_max_parallel_tasks_per_type_ratio`
|
|
26
|
+
* default). Mirrored here only so the derivation test can assert the cap
|
|
27
|
+
* stays >= 2 for every supported instance type. */
|
|
28
|
+
export declare const SCHEDULE_POOL_PER_TYPE_CAP_RATIO = 0.8;
|
|
29
|
+
/**
|
|
30
|
+
* `max_server_memory_usage_to_ram_ratio` rendered into the server config.
|
|
31
|
+
* 0.75 gives the kernel a cushion on the cgroup for CH's untracked
|
|
32
|
+
* allocations (libc allocator, executable image, malloc fragmentation,
|
|
33
|
+
* jemalloc virtual reservations on ARM64); the default 0.9 left ~300 MB on
|
|
34
|
+
* the 4 GiB host and OOM-killed the process at boot (exit 137). Exported so
|
|
35
|
+
* the cache-budget test computes against the same ratio the XML ships —
|
|
36
|
+
* re-tuning it here moves both sites together.
|
|
37
|
+
*/
|
|
38
|
+
export declare const MAX_SERVER_MEMORY_RAM_RATIO = 0.75;
|
|
39
|
+
/** Retention for every configured system log table. One policy, eight
|
|
40
|
+
* `<ttl>` sites in the rendered config — interpolated so a retention
|
|
41
|
+
* change is a single edit. */
|
|
42
|
+
export declare const SYSTEM_LOG_TTL = "event_date + INTERVAL 14 DAY DELETE";
|
|
43
|
+
/**
|
|
44
|
+
* Derive the server-level pool/cache/concurrency configuration from the
|
|
45
|
+
* instance type. Replaces the hand-tuned m7g.medium constants that silently
|
|
46
|
+
* stopped fitting when the instance type moved (the 2026-08-03 r8g.medium
|
|
47
|
+
* flip scaled only the container memory limit; every pool and cache stayed
|
|
48
|
+
* at 4 GiB values until the 2026-08-12 prod investigation caught it).
|
|
49
|
+
*/
|
|
50
|
+
export declare function deriveClickHouseServerTuning(instanceType: string): ClickHouseServerTuning;
|
|
51
|
+
/**
|
|
52
|
+
* Derive the four workload-class default profiles from the instance type.
|
|
53
|
+
* Memory caps, external-spill thresholds and row-read ceilings scale with
|
|
54
|
+
* instance memory; thread caps scale with vCPUs. At the default
|
|
55
|
+
* m7g.medium this reproduces the historical hand-tuned values exactly
|
|
56
|
+
* (pinned by clickhouseTuning.test.ts).
|
|
57
|
+
*
|
|
58
|
+
* `high_throughput_ingest.maxConcurrentQueriesForUser` deliberately does
|
|
59
|
+
* NOT scale: it mirrors the webapp's client-side FIFO gate
|
|
60
|
+
* (`webapp/app/.server/lib/clickhouse/appQuerySlots.ts`) — a users.xml
|
|
61
|
+
* parity test on the webapp side enforces the pair, so the value is a
|
|
62
|
+
* client contract, not an instance-capacity knob.
|
|
63
|
+
*/
|
|
64
|
+
export declare function deriveClickHouseDefaultProfiles(instanceType: string): Readonly<Record<string, Readonly<ProfileSpec>>>;
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
import { CLICKHOUSE_INSTANCE_SPECS } from "./clickhouseConstants.js";
|
|
2
|
+
/** 4 GiB / 1 vCPU (m7g.medium) baseline the scaling formulas anchor to.
|
|
3
|
+
* These are the proven prod values the construct shipped hand-tuned before
|
|
4
|
+
* derivation existed — `deriveClickHouseServerTuning("m7g.medium")` must
|
|
5
|
+
* reproduce them exactly (pinned by clickhouseTuning.test.ts), with the one
|
|
6
|
+
* deliberate exception of the schedule pool (see SCHEDULE_POOL_FLOOR). */
|
|
7
|
+
const BASELINE_MEMORY_GIB = 4;
|
|
8
|
+
const BASE_MARK_CACHE_BYTES = 402653184; // 384 MiB
|
|
9
|
+
const BASE_INDEX_MARK_CACHE_BYTES = 134217728; // 128 MiB
|
|
10
|
+
const BASE_QUERY_CACHE_BYTES = 268435456; // 256 MiB
|
|
11
|
+
const BASE_MAX_CONCURRENT_QUERIES_PER_VCPU = 8;
|
|
12
|
+
/**
|
|
13
|
+
* Floor on `background_schedule_pool_size`, independent of instance size.
|
|
14
|
+
*
|
|
15
|
+
* ClickHouse >= 25.9 (upstream PR #84008) caps same-type concurrent
|
|
16
|
+
* background tasks at `floor(pool_size * 0.8)` and logs
|
|
17
|
+
* `<Warning> BgSchPool: Temporarily pause scheduling of tasks with id ...`
|
|
18
|
+
* every time the cap defers a task. The pre-derivation hand-tuned pool of 2
|
|
19
|
+
* produced a cap of 1 — the warning fired on essentially every task start
|
|
20
|
+
* (~35k lines/hour observed on prod, 2026-08-12) and serialised ALL
|
|
21
|
+
* background scheduling (merge selection, TTL moves, parts cleanup, S3 blob
|
|
22
|
+
* deletion) through two threads. Schedule-pool threads are dispatchers, not
|
|
23
|
+
* workers — they spend their lives sleeping between ticks — so the floor
|
|
24
|
+
* costs thread stacks, not CPU. 8 gives a per-type cap of 6.
|
|
25
|
+
*
|
|
26
|
+
* The cap is computed ONCE at pool creation: raising this value requires a
|
|
27
|
+
* server restart, not a config reload.
|
|
28
|
+
*/
|
|
29
|
+
const SCHEDULE_POOL_FLOOR = 8;
|
|
30
|
+
/** Ratio ClickHouse >= 25.9 applies to the schedule pool to derive the
|
|
31
|
+
* per-type concurrent-task cap (`background_schedule_pool_max_parallel_tasks_per_type_ratio`
|
|
32
|
+
* default). Mirrored here only so the derivation test can assert the cap
|
|
33
|
+
* stays >= 2 for every supported instance type. */
|
|
34
|
+
export const SCHEDULE_POOL_PER_TYPE_CAP_RATIO = 0.8;
|
|
35
|
+
/**
|
|
36
|
+
* `max_server_memory_usage_to_ram_ratio` rendered into the server config.
|
|
37
|
+
* 0.75 gives the kernel a cushion on the cgroup for CH's untracked
|
|
38
|
+
* allocations (libc allocator, executable image, malloc fragmentation,
|
|
39
|
+
* jemalloc virtual reservations on ARM64); the default 0.9 left ~300 MB on
|
|
40
|
+
* the 4 GiB host and OOM-killed the process at boot (exit 137). Exported so
|
|
41
|
+
* the cache-budget test computes against the same ratio the XML ships —
|
|
42
|
+
* re-tuning it here moves both sites together.
|
|
43
|
+
*/
|
|
44
|
+
export const MAX_SERVER_MEMORY_RAM_RATIO = 0.75;
|
|
45
|
+
/** Retention for every configured system log table. One policy, eight
|
|
46
|
+
* `<ttl>` sites in the rendered config — interpolated so a retention
|
|
47
|
+
* change is a single edit. */
|
|
48
|
+
export const SYSTEM_LOG_TTL = "event_date + INTERVAL 14 DAY DELETE";
|
|
49
|
+
function resolveSpec(instanceType) {
|
|
50
|
+
const spec = CLICKHOUSE_INSTANCE_SPECS[instanceType];
|
|
51
|
+
if (spec === undefined) {
|
|
52
|
+
throw new Error(`ClickHouseDatabase: unknown instance type '${instanceType}' — cannot ` +
|
|
53
|
+
"derive server tuning. Supported: " +
|
|
54
|
+
`${Object.keys(CLICKHOUSE_INSTANCE_SPECS).join(", ")}. Add the ` +
|
|
55
|
+
"type's spec to CLICKHOUSE_INSTANCE_SPECS (clickhouseConstants.ts) " +
|
|
56
|
+
"to support it.");
|
|
57
|
+
}
|
|
58
|
+
return spec;
|
|
59
|
+
}
|
|
60
|
+
/** Linear memory scale relative to the 4 GiB baseline. Every supported spec
|
|
61
|
+
* is a power-of-two multiple of 4 GiB, so the scale is always an integer
|
|
62
|
+
* and every derived byte value stays exact. */
|
|
63
|
+
function memoryScale(spec) {
|
|
64
|
+
return spec.memoryGib / BASELINE_MEMORY_GIB;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Derive the server-level pool/cache/concurrency configuration from the
|
|
68
|
+
* instance type. Replaces the hand-tuned m7g.medium constants that silently
|
|
69
|
+
* stopped fitting when the instance type moved (the 2026-08-03 r8g.medium
|
|
70
|
+
* flip scaled only the container memory limit; every pool and cache stayed
|
|
71
|
+
* at 4 GiB values until the 2026-08-12 prod investigation caught it).
|
|
72
|
+
*/
|
|
73
|
+
export function deriveClickHouseServerTuning(instanceType) {
|
|
74
|
+
const spec = resolveSpec(instanceType);
|
|
75
|
+
const scale = memoryScale(spec);
|
|
76
|
+
// Merge/mutation workers burn CPU — scale with vCPUs, floor at the proven
|
|
77
|
+
// 1-vCPU value. The ratio then allows 2x slots for merge SELECTION, while
|
|
78
|
+
// actual execution stays bounded by the pool.
|
|
79
|
+
const backgroundPoolSize = Math.max(2, 2 * spec.vcpus);
|
|
80
|
+
const mergesMutationsConcurrencyRatio = 2;
|
|
81
|
+
// ClickHouse sanity-checks the merge_tree free-entry thresholds against
|
|
82
|
+
// pool * ratio at startup (BAD_ARGUMENTS when a threshold exceeds the
|
|
83
|
+
// slot count). Deriving from the slot count keeps the invariant by
|
|
84
|
+
// construction: slots/4 approximates the proportion the CH defaults
|
|
85
|
+
// (20/8/25 against 32 slots) express, floored at 1 so small pools can
|
|
86
|
+
// always start a merge/mutation the moment a slot frees.
|
|
87
|
+
const slots = backgroundPoolSize * mergesMutationsConcurrencyRatio;
|
|
88
|
+
const freeEntries = Math.max(1, Math.floor(slots / 4));
|
|
89
|
+
return {
|
|
90
|
+
// Slots time-slice rather than parallelise on small hosts; 8 per vCPU is
|
|
91
|
+
// the proven m7g.medium value (raised 4→8 2026-07-28 for the webapp
|
|
92
|
+
// overview's 6-10 queries/page fan-out).
|
|
93
|
+
maxConcurrentQueries: BASE_MAX_CONCURRENT_QUERIES_PER_VCPU * spec.vcpus,
|
|
94
|
+
backgroundPoolSize,
|
|
95
|
+
backgroundSchedulePoolSize: Math.max(SCHEDULE_POOL_FLOOR, 4 * spec.vcpus),
|
|
96
|
+
// Moves are I/O-bound (S3 cold-tier uploads); fetches serve replication
|
|
97
|
+
// (none in this single-node design) and the broker pool serves Kafka
|
|
98
|
+
// (none) — both stay minimal at every size.
|
|
99
|
+
backgroundMovePoolSize: spec.vcpus >= 4 ? 2 : 1,
|
|
100
|
+
backgroundFetchesPoolSize: 1,
|
|
101
|
+
backgroundMessageBrokerSchedulePoolSize: 1,
|
|
102
|
+
mergesMutationsConcurrencyRatio,
|
|
103
|
+
freeEntriesToExecuteMutation: freeEntries,
|
|
104
|
+
freeEntriesToLowerMaxSizeOfMerge: freeEntries,
|
|
105
|
+
freeEntriesToExecuteOptimizeEntirePartition: freeEntries,
|
|
106
|
+
markCacheBytes: BASE_MARK_CACHE_BYTES * scale,
|
|
107
|
+
indexMarkCacheBytes: BASE_INDEX_MARK_CACHE_BYTES * scale,
|
|
108
|
+
queryCacheBytes: BASE_QUERY_CACHE_BYTES * scale
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Derive the four workload-class default profiles from the instance type.
|
|
113
|
+
* Memory caps, external-spill thresholds and row-read ceilings scale with
|
|
114
|
+
* instance memory; thread caps scale with vCPUs. At the default
|
|
115
|
+
* m7g.medium this reproduces the historical hand-tuned values exactly
|
|
116
|
+
* (pinned by clickhouseTuning.test.ts).
|
|
117
|
+
*
|
|
118
|
+
* `high_throughput_ingest.maxConcurrentQueriesForUser` deliberately does
|
|
119
|
+
* NOT scale: it mirrors the webapp's client-side FIFO gate
|
|
120
|
+
* (`webapp/app/.server/lib/clickhouse/appQuerySlots.ts`) — a users.xml
|
|
121
|
+
* parity test on the webapp side enforces the pair, so the value is a
|
|
122
|
+
* client contract, not an instance-capacity knob.
|
|
123
|
+
*/
|
|
124
|
+
export function deriveClickHouseDefaultProfiles(instanceType) {
|
|
125
|
+
const spec = resolveSpec(instanceType);
|
|
126
|
+
const scale = memoryScale(spec);
|
|
127
|
+
return Object.freeze({
|
|
128
|
+
high_throughput_ingest: Object.freeze({
|
|
129
|
+
maxThreads: spec.vcpus,
|
|
130
|
+
maxInsertThreads: spec.vcpus,
|
|
131
|
+
maxConcurrentQueriesForUser: 6,
|
|
132
|
+
logQueriesMinQueryDurationMs: 100,
|
|
133
|
+
optimizeMoveToPrewhere: true,
|
|
134
|
+
useQueryConditionCache: true,
|
|
135
|
+
useSkipIndexesIfFinal: true,
|
|
136
|
+
applyRowPolicyAfterFinal: true,
|
|
137
|
+
applyPrewhereAfterFinal: true,
|
|
138
|
+
doNotMergeAcrossPartitionsSelectFinal: true,
|
|
139
|
+
asyncInsert: true,
|
|
140
|
+
waitForAsyncInsert: true,
|
|
141
|
+
asyncInsertMaxDataSize: 10000000,
|
|
142
|
+
asyncInsertBusyTimeoutMinMs: 50,
|
|
143
|
+
asyncInsertBusyTimeoutMaxMs: 2000,
|
|
144
|
+
asyncInsertUseAdaptiveBusyTimeout: true,
|
|
145
|
+
asyncInsertDeduplicate: true,
|
|
146
|
+
inputFormatParallelParsing: false,
|
|
147
|
+
outputFormatParallelFormatting: false,
|
|
148
|
+
queryPlanOptimizeLazyMaterialization: true,
|
|
149
|
+
maxMemoryUsage: 1610612736 * scale, // 1.5 GiB baseline
|
|
150
|
+
maxMemoryUsageForUser: 2147483648 * scale, // 2 GiB baseline
|
|
151
|
+
maxBytesBeforeExternalSort: 536870912 * scale, // 512 MiB baseline
|
|
152
|
+
maxBytesBeforeExternalGroupBy: 536870912 * scale,
|
|
153
|
+
maxExecutionTime: 30,
|
|
154
|
+
// The ceiling an unbounded consumer query dies against (TOO_MANY_ROWS).
|
|
155
|
+
// Scales with memory so a bigger node's tables get proportionate
|
|
156
|
+
// headroom — but consumers must still time-bound their scans; this is
|
|
157
|
+
// a guard rail, not a budget to grow into.
|
|
158
|
+
maxRowsToRead: 10000000 * scale
|
|
159
|
+
}),
|
|
160
|
+
audit_append: Object.freeze({
|
|
161
|
+
maxThreads: 1,
|
|
162
|
+
maxInsertThreads: 1,
|
|
163
|
+
maxConcurrentQueriesForUser: 2,
|
|
164
|
+
maxMemoryUsage: 500000000 * scale,
|
|
165
|
+
maxExecutionTime: 10,
|
|
166
|
+
asyncInsert: true,
|
|
167
|
+
waitForAsyncInsert: true
|
|
168
|
+
}),
|
|
169
|
+
read_only: Object.freeze({
|
|
170
|
+
maxThreads: spec.vcpus,
|
|
171
|
+
maxConcurrentQueriesForUser: 1,
|
|
172
|
+
maxMemoryUsage: 1000000000 * scale,
|
|
173
|
+
maxExecutionTime: 3600
|
|
174
|
+
}),
|
|
175
|
+
ddl_admin: Object.freeze({
|
|
176
|
+
maxThreads: spec.vcpus,
|
|
177
|
+
maxConcurrentQueriesForUser: 1,
|
|
178
|
+
maxMemoryUsage: 1000000000 * scale,
|
|
179
|
+
maxExecutionTime: 1800
|
|
180
|
+
})
|
|
181
|
+
});
|
|
182
|
+
}
|
|
@@ -11,6 +11,13 @@ export interface BuildClickHouseUserDataOptions {
|
|
|
11
11
|
backupBucketName: string;
|
|
12
12
|
/** AWS region of the backup bucket. */
|
|
13
13
|
backupBucketRegion: string;
|
|
14
|
+
/**
|
|
15
|
+
* Resolved EC2 instance type. Server pools, caches and concurrency caps
|
|
16
|
+
* derive from it via `deriveClickHouseServerTuning` — must be a key of
|
|
17
|
+
* `CLICKHOUSE_INSTANCE_SPECS` (unknown types throw at synth, same contract
|
|
18
|
+
* as `clickHouseTaskMemoryMiB`).
|
|
19
|
+
*/
|
|
20
|
+
instanceType: string;
|
|
14
21
|
/**
|
|
15
22
|
* Optional S3 cold-tier configuration.
|
|
16
23
|
* When omitted, single-tier hot storage on the EC2 EBS volume is used.
|