@fjall/components-infrastructure 10.1.2 → 11.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,19 +10,17 @@ export const CLICKHOUSE_DATABASE_NAME = "analytics";
10
10
  * of 2 vCPU) for the rest of the hour. Graviton3 is also ~25% faster on
11
11
  * columnar scans than Graviton2.
12
12
  *
13
- * Settings are tuned for 1 vCPU sustained (max_threads=1 across all profiles,
14
- * max_concurrent_queries=8) — see clickhouseUserData.ts. Bumping to a larger
15
- * instance MUST be paired with raising those thread/concurrency caps.
16
- *
17
13
  * Next size up (2026-07-31 right-sizing, designs/2026-07-31-clickhouse-
18
14
  * factory-fit-for-purpose.md): `r8g.medium` (1 vCPU Graviton4, 8 GiB,
19
15
  * ~+30%/core over Graviton3) — CH's own doctrine floors production at
20
16
  * 8 GiB, and the memory-bound single-tenant profile wants RAM before
21
17
  * cores. `m7g.large` (2 vCPU, 8 GiB) only when CPU-bound. Set via
22
18
  * `clickhouseInstanceType` CDK context or the `instanceType` prop —
23
- * the container memory limit derives automatically via
24
- * `clickHouseTaskMemoryMiB()`, but the thread/concurrency caps in
25
- * clickhouseUserData.ts still need a lockstep re-tune. */
19
+ * the container memory limit, server pools/caches, and profile resource
20
+ * caps ALL derive from the type automatically (`clickHouseTaskMemoryMiB()`,
21
+ * `deriveClickHouseServerTuning()`, `deriveClickHouseDefaultProfiles()`).
22
+ * A size change needs no manual re-tune — but it DOES need an instance
23
+ * refresh + server restart to take effect (config is baked into user-data). */
26
24
  export const DEFAULT_CLICKHOUSE_INSTANCE_TYPE = "m7g.medium";
27
25
  /** ClickHouse container image. Explicit `docker.io/` prefix is required so
28
26
  * string-form consumers in `ecsImages.ts#getContainerImage()` route through
@@ -62,23 +60,26 @@ export const CLICKHOUSE_HOST_RESERVE_MIB = 1024;
62
60
  * derived from nominal memory, so the host reserve is what absorbs this
63
61
  * withholding at task placement. */
64
62
  export const CLICKHOUSE_ECS_RESERVED_MEMORY_MIB = 256;
65
- /** Total memory (GiB) of the instance types the ClickHouse construct knows
66
- * how to size a container for. Values are the AWS nominal figures. */
67
- export const CLICKHOUSE_INSTANCE_MEMORY_GIB = {
68
- "t4g.medium": 4,
69
- "m7g.medium": 4,
70
- "m8g.medium": 4,
71
- "r7g.medium": 8,
72
- "r8g.medium": 8,
73
- "m7g.large": 8,
74
- "m8g.large": 8,
75
- "r7g.large": 16,
76
- "r8g.large": 16,
77
- "m7g.xlarge": 16,
78
- "m8g.xlarge": 16,
79
- "r7g.xlarge": 32,
80
- "r8g.xlarge": 32
63
+ export const CLICKHOUSE_INSTANCE_SPECS = {
64
+ "t4g.medium": { vcpus: 2, memoryGib: 4 },
65
+ "m7g.medium": { vcpus: 1, memoryGib: 4 },
66
+ "m8g.medium": { vcpus: 1, memoryGib: 4 },
67
+ "r7g.medium": { vcpus: 1, memoryGib: 8 },
68
+ "r8g.medium": { vcpus: 1, memoryGib: 8 },
69
+ "m7g.large": { vcpus: 2, memoryGib: 8 },
70
+ "m8g.large": { vcpus: 2, memoryGib: 8 },
71
+ "r7g.large": { vcpus: 2, memoryGib: 16 },
72
+ "r8g.large": { vcpus: 2, memoryGib: 16 },
73
+ "m7g.xlarge": { vcpus: 4, memoryGib: 16 },
74
+ "m8g.xlarge": { vcpus: 4, memoryGib: 16 },
75
+ "r7g.xlarge": { vcpus: 4, memoryGib: 32 },
76
+ "r8g.xlarge": { vcpus: 4, memoryGib: 32 }
81
77
  };
78
+ /** Derived memory-only view of CLICKHOUSE_INSTANCE_SPECS. */
79
+ export const CLICKHOUSE_INSTANCE_MEMORY_GIB = Object.fromEntries(Object.entries(CLICKHOUSE_INSTANCE_SPECS).map(([type, spec]) => [
80
+ type,
81
+ spec.memoryGib
82
+ ]));
82
83
  /** ECS container memory for the ClickHouse server task, derived from the
83
84
  * instance type so a size change cannot leave the hand-set container cap
84
85
  * behind (the pre-2026-07 shape: a hardcoded 3072 that silently starved —
@@ -241,6 +242,14 @@ export const CLICKHOUSE_CLOUDMAP_SERVICE_NAME = "clickhouse";
241
242
  * live-reload path into a silent no-op without any compile/test signal,
242
243
  * so this lives as a single source of truth. */
243
244
  export const CLICKHOUSE_SERVER_CONTAINER_NAME = "clickhouse";
245
+ /** ECS service name for the ClickHouse server. Coupled cross-repo value:
246
+ * the webapp log collector excludes this service from customer log
247
+ * collection by exact name (`CLICKHOUSE_SERVER_SERVICE_NAME` at
248
+ * `webapp/app/.server/services/monitoring/logGroupResolution.ts`) —
249
+ * deliberately duplicated rather than shared, so a webapp deploy is never
250
+ * release-coupled to a constructs mint. Renaming this service breaks that
251
+ * exclusion silently; update both sites together. */
252
+ export const CLICKHOUSE_SERVICE_NAME = "ClickHouseService";
244
253
  /** Materialised views that benefit from periodic OPTIMIZE to reduce part count at read time.
245
254
  * These are not ReplacingMergeTree (no dedup needed) but un-merged parts force
246
255
  * read-time aggregation which degrades query performance.
@@ -68,4 +68,24 @@ export type ProfileSpec = z.infer<typeof ProfileSpecSchema>;
68
68
  * XML element names with no escaping.
69
69
  */
70
70
  export declare const PROFILE_NAME_PATTERN: RegExp;
71
+ /**
72
+ * Four workload-class default profiles at the DEFAULT instance type's scale.
73
+ * Since the 2026-08-12 instance-scaled derivation the canonical source is
74
+ * `deriveClickHouseDefaultProfiles(instanceType)` in `clickhouseTuning.ts` —
75
+ * the construct derives its default `profiles:` from the RESOLVED instance
76
+ * type, so this constant serves only consumers with no instance in hand
77
+ * (parity tests, docs). At `DEFAULT_CLICKHOUSE_INSTANCE_TYPE` the derivation
78
+ * reproduces the historical hand-tuned values byte-for-byte (pinned by
79
+ * clickhouseTuning.test.ts).
80
+ *
81
+ * `high_throughput_ingest.maxConcurrentQueriesForUser` (6) mirrors the
82
+ * webapp's client-side FIFO gate
83
+ * (`webapp/app/.server/lib/clickhouse/appQuerySlots.ts`) — a users.xml
84
+ * parity test enforces the pair, and the derivation deliberately never
85
+ * scales it.
86
+ *
87
+ * Frozen via `Object.freeze` on the outer record AND each sub-object so a
88
+ * consumer cannot mutate the defaults in place. Pass your own `profiles:`
89
+ * map to the construct to extend / override; never mutate this constant.
90
+ */
71
91
  export declare const ClickHouseDefaultProfiles: Readonly<Record<string, Readonly<ProfileSpec>>>;
@@ -1,4 +1,6 @@
1
1
  import { z } from "zod";
2
+ import { DEFAULT_CLICKHOUSE_INSTANCE_TYPE } from "./clickhouseConstants.js";
3
+ import { deriveClickHouseDefaultProfiles } from "./clickhouseTuning.js";
2
4
  /** ClickHouse user/profile name pattern — lowercase snake_case, leading
3
5
  * letter only. Shared by `ClickHouseSchemaAdminSchema.name`, the per-profile
4
6
  * key regex applied inside `ClickHouseDatabase`'s construct-time validation
@@ -86,82 +88,23 @@ export const ProfileSpecSchema = z
86
88
  */
87
89
  export const PROFILE_NAME_PATTERN = NAME_PATTERN;
88
90
  /**
89
- * Four workload-class default profiles. Cap values lifted byte-for-byte
90
- * from the pre-Phase-4b inline `<app_writer>` / `<audit_writer>` /
91
- * `<backup_reader>` / `<schema_admin>` blocks at
92
- * `clickhouseUserData.ts:296-382` zero behavioural drift from the
93
- * pre-4b prod resource caps, EXCEPT high_throughput_ingest's
94
- * `maxConcurrentQueriesForUser`, deliberately raised 2→6 (2026-07-28):
95
- * the webapp overview fires 6–10 queries per page load and the cap of 2
96
- * threw TOO_MANY_SIMULTANEOUS_QUERIES instead of queueing (CH does not
97
- * queue past this cap — `queue_max_wait_ms` empirically does not apply).
98
- * Consumers pair it with a client-side FIFO gate
99
- * (`webapp/app/.server/lib/clickhouse/appQuerySlots.ts`) whose limit
100
- * MUST mirror this value — a users.xml parity test enforces the pair.
91
+ * Four workload-class default profiles at the DEFAULT instance type's scale.
92
+ * Since the 2026-08-12 instance-scaled derivation the canonical source is
93
+ * `deriveClickHouseDefaultProfiles(instanceType)` in `clickhouseTuning.ts`
94
+ * the construct derives its default `profiles:` from the RESOLVED instance
95
+ * type, so this constant serves only consumers with no instance in hand
96
+ * (parity tests, docs). At `DEFAULT_CLICKHOUSE_INSTANCE_TYPE` the derivation
97
+ * reproduces the historical hand-tuned values byte-for-byte (pinned by
98
+ * clickhouseTuning.test.ts).
101
99
  *
102
- * Frozen via `Object.freeze` on the outer record AND each sub-object so
103
- * a consumer cannot mutate the defaults in place. Pass your own
104
- * `profiles:` map to the construct to extend / override; never mutate
105
- * this constant.
100
+ * `high_throughput_ingest.maxConcurrentQueriesForUser` (6) mirrors the
101
+ * webapp's client-side FIFO gate
102
+ * (`webapp/app/.server/lib/clickhouse/appQuerySlots.ts`) a users.xml
103
+ * parity test enforces the pair, and the derivation deliberately never
104
+ * scales it.
105
+ *
106
+ * Frozen via `Object.freeze` on the outer record AND each sub-object so a
107
+ * consumer cannot mutate the defaults in place. Pass your own `profiles:`
108
+ * map to the construct to extend / override; never mutate this constant.
106
109
  */
107
- const _ClickHouseDefaultProfilesRaw = {
108
- high_throughput_ingest: {
109
- maxThreads: 1,
110
- maxInsertThreads: 1,
111
- maxConcurrentQueriesForUser: 6,
112
- logQueriesMinQueryDurationMs: 100,
113
- optimizeMoveToPrewhere: true,
114
- useQueryConditionCache: true,
115
- useSkipIndexesIfFinal: true,
116
- applyRowPolicyAfterFinal: true,
117
- applyPrewhereAfterFinal: true,
118
- doNotMergeAcrossPartitionsSelectFinal: true,
119
- asyncInsert: true,
120
- waitForAsyncInsert: true,
121
- asyncInsertMaxDataSize: 10000000,
122
- asyncInsertBusyTimeoutMinMs: 50,
123
- asyncInsertBusyTimeoutMaxMs: 2000,
124
- asyncInsertUseAdaptiveBusyTimeout: true,
125
- asyncInsertDeduplicate: true,
126
- inputFormatParallelParsing: false,
127
- outputFormatParallelFormatting: false,
128
- queryPlanOptimizeLazyMaterialization: true,
129
- maxMemoryUsage: 1610612736,
130
- maxMemoryUsageForUser: 2147483648,
131
- maxBytesBeforeExternalSort: 536870912,
132
- maxBytesBeforeExternalGroupBy: 536870912,
133
- maxExecutionTime: 30,
134
- maxRowsToRead: 10000000
135
- },
136
- audit_append: {
137
- maxThreads: 1,
138
- maxInsertThreads: 1,
139
- maxConcurrentQueriesForUser: 2,
140
- maxMemoryUsage: 500000000,
141
- maxExecutionTime: 10,
142
- asyncInsert: true,
143
- waitForAsyncInsert: true
144
- },
145
- read_only: {
146
- maxThreads: 1,
147
- maxConcurrentQueriesForUser: 1,
148
- maxMemoryUsage: 1000000000,
149
- maxExecutionTime: 3600
150
- },
151
- ddl_admin: {
152
- maxThreads: 1,
153
- maxConcurrentQueriesForUser: 1,
154
- maxMemoryUsage: 1000000000,
155
- maxExecutionTime: 1800
156
- }
157
- };
158
- export const ClickHouseDefaultProfiles = Object.freeze({
159
- high_throughput_ingest: Object.freeze({
160
- ..._ClickHouseDefaultProfilesRaw.high_throughput_ingest
161
- }),
162
- audit_append: Object.freeze({
163
- ..._ClickHouseDefaultProfilesRaw.audit_append
164
- }),
165
- read_only: Object.freeze({ ..._ClickHouseDefaultProfilesRaw.read_only }),
166
- ddl_admin: Object.freeze({ ..._ClickHouseDefaultProfilesRaw.ddl_admin })
167
- });
110
+ export const ClickHouseDefaultProfiles = deriveClickHouseDefaultProfiles(DEFAULT_CLICKHOUSE_INSTANCE_TYPE);
@@ -0,0 +1,64 @@
1
+ import type { ProfileSpec } from "./clickhouseSchemas.js";
2
+ /**
3
+ * Server-level tuning values derived from the instance type. Consumed by
4
+ * `generateServerConfigXml` — every value here lands in the rendered
5
+ * `config.d/fjall.xml`, so a change means a launch-template refresh +
6
+ * instance refresh + server restart to take effect (the schedule-pool
7
+ * per-type cap in particular is computed once at pool creation).
8
+ */
9
+ export interface ClickHouseServerTuning {
10
+ maxConcurrentQueries: number;
11
+ backgroundPoolSize: number;
12
+ backgroundSchedulePoolSize: number;
13
+ backgroundMovePoolSize: number;
14
+ backgroundFetchesPoolSize: number;
15
+ backgroundMessageBrokerSchedulePoolSize: number;
16
+ mergesMutationsConcurrencyRatio: number;
17
+ freeEntriesToExecuteMutation: number;
18
+ freeEntriesToLowerMaxSizeOfMerge: number;
19
+ freeEntriesToExecuteOptimizeEntirePartition: number;
20
+ markCacheBytes: number;
21
+ indexMarkCacheBytes: number;
22
+ queryCacheBytes: number;
23
+ }
24
+ /** Ratio ClickHouse >= 25.9 applies to the schedule pool to derive the
25
+ * per-type concurrent-task cap (`background_schedule_pool_max_parallel_tasks_per_type_ratio`
26
+ * default). Mirrored here only so the derivation test can assert the cap
27
+ * stays >= 2 for every supported instance type. */
28
+ export declare const SCHEDULE_POOL_PER_TYPE_CAP_RATIO = 0.8;
29
+ /**
30
+ * `max_server_memory_usage_to_ram_ratio` rendered into the server config.
31
+ * 0.75 gives the kernel a cushion on the cgroup for CH's untracked
32
+ * allocations (libc allocator, executable image, malloc fragmentation,
33
+ * jemalloc virtual reservations on ARM64); the default 0.9 left ~300 MB on
34
+ * the 4 GiB host and OOM-killed the process at boot (exit 137). Exported so
35
+ * the cache-budget test computes against the same ratio the XML ships —
36
+ * re-tuning it here moves both sites together.
37
+ */
38
+ export declare const MAX_SERVER_MEMORY_RAM_RATIO = 0.75;
39
+ /** Retention for every configured system log table. One policy, eight
40
+ * `<ttl>` sites in the rendered config — interpolated so a retention
41
+ * change is a single edit. */
42
+ export declare const SYSTEM_LOG_TTL = "event_date + INTERVAL 14 DAY DELETE";
43
+ /**
44
+ * Derive the server-level pool/cache/concurrency configuration from the
45
+ * instance type. Replaces the hand-tuned m7g.medium constants that silently
46
+ * stopped fitting when the instance type moved (the 2026-08-03 r8g.medium
47
+ * flip scaled only the container memory limit; every pool and cache stayed
48
+ * at 4 GiB values until the 2026-08-12 prod investigation caught it).
49
+ */
50
+ export declare function deriveClickHouseServerTuning(instanceType: string): ClickHouseServerTuning;
51
+ /**
52
+ * Derive the four workload-class default profiles from the instance type.
53
+ * Memory caps, external-spill thresholds and row-read ceilings scale with
54
+ * instance memory; thread caps scale with vCPUs. At the default
55
+ * m7g.medium this reproduces the historical hand-tuned values exactly
56
+ * (pinned by clickhouseTuning.test.ts).
57
+ *
58
+ * `high_throughput_ingest.maxConcurrentQueriesForUser` deliberately does
59
+ * NOT scale: it mirrors the webapp's client-side FIFO gate
60
+ * (`webapp/app/.server/lib/clickhouse/appQuerySlots.ts`) — a users.xml
61
+ * parity test on the webapp side enforces the pair, so the value is a
62
+ * client contract, not an instance-capacity knob.
63
+ */
64
+ export declare function deriveClickHouseDefaultProfiles(instanceType: string): Readonly<Record<string, Readonly<ProfileSpec>>>;
@@ -0,0 +1,182 @@
1
+ import { CLICKHOUSE_INSTANCE_SPECS } from "./clickhouseConstants.js";
2
+ /** 4 GiB / 1 vCPU (m7g.medium) baseline the scaling formulas anchor to.
3
+ * These are the proven prod values the construct shipped hand-tuned before
4
+ * derivation existed — `deriveClickHouseServerTuning("m7g.medium")` must
5
+ * reproduce them exactly (pinned by clickhouseTuning.test.ts), with the one
6
+ * deliberate exception of the schedule pool (see SCHEDULE_POOL_FLOOR). */
7
+ const BASELINE_MEMORY_GIB = 4;
8
+ const BASE_MARK_CACHE_BYTES = 402653184; // 384 MiB
9
+ const BASE_INDEX_MARK_CACHE_BYTES = 134217728; // 128 MiB
10
+ const BASE_QUERY_CACHE_BYTES = 268435456; // 256 MiB
11
+ const BASE_MAX_CONCURRENT_QUERIES_PER_VCPU = 8;
12
+ /**
13
+ * Floor on `background_schedule_pool_size`, independent of instance size.
14
+ *
15
+ * ClickHouse >= 25.9 (upstream PR #84008) caps same-type concurrent
16
+ * background tasks at `floor(pool_size * 0.8)` and logs
17
+ * `<Warning> BgSchPool: Temporarily pause scheduling of tasks with id ...`
18
+ * every time the cap defers a task. The pre-derivation hand-tuned pool of 2
19
+ * produced a cap of 1 — the warning fired on essentially every task start
20
+ * (~35k lines/hour observed on prod, 2026-08-12) and serialised ALL
21
+ * background scheduling (merge selection, TTL moves, parts cleanup, S3 blob
22
+ * deletion) through two threads. Schedule-pool threads are dispatchers, not
23
+ * workers — they spend their lives sleeping between ticks — so the floor
24
+ * costs thread stacks, not CPU. 8 gives a per-type cap of 6.
25
+ *
26
+ * The cap is computed ONCE at pool creation: raising this value requires a
27
+ * server restart, not a config reload.
28
+ */
29
+ const SCHEDULE_POOL_FLOOR = 8;
30
+ /** Ratio ClickHouse >= 25.9 applies to the schedule pool to derive the
31
+ * per-type concurrent-task cap (`background_schedule_pool_max_parallel_tasks_per_type_ratio`
32
+ * default). Mirrored here only so the derivation test can assert the cap
33
+ * stays >= 2 for every supported instance type. */
34
+ export const SCHEDULE_POOL_PER_TYPE_CAP_RATIO = 0.8;
35
+ /**
36
+ * `max_server_memory_usage_to_ram_ratio` rendered into the server config.
37
+ * 0.75 gives the kernel a cushion on the cgroup for CH's untracked
38
+ * allocations (libc allocator, executable image, malloc fragmentation,
39
+ * jemalloc virtual reservations on ARM64); the default 0.9 left ~300 MB on
40
+ * the 4 GiB host and OOM-killed the process at boot (exit 137). Exported so
41
+ * the cache-budget test computes against the same ratio the XML ships —
42
+ * re-tuning it here moves both sites together.
43
+ */
44
+ export const MAX_SERVER_MEMORY_RAM_RATIO = 0.75;
45
+ /** Retention for every configured system log table. One policy, eight
46
+ * `<ttl>` sites in the rendered config — interpolated so a retention
47
+ * change is a single edit. */
48
+ export const SYSTEM_LOG_TTL = "event_date + INTERVAL 14 DAY DELETE";
49
+ function resolveSpec(instanceType) {
50
+ const spec = CLICKHOUSE_INSTANCE_SPECS[instanceType];
51
+ if (spec === undefined) {
52
+ throw new Error(`ClickHouseDatabase: unknown instance type '${instanceType}' — cannot ` +
53
+ "derive server tuning. Supported: " +
54
+ `${Object.keys(CLICKHOUSE_INSTANCE_SPECS).join(", ")}. Add the ` +
55
+ "type's spec to CLICKHOUSE_INSTANCE_SPECS (clickhouseConstants.ts) " +
56
+ "to support it.");
57
+ }
58
+ return spec;
59
+ }
60
+ /** Linear memory scale relative to the 4 GiB baseline. Every supported spec
61
+ * is a power-of-two multiple of 4 GiB, so the scale is always an integer
62
+ * and every derived byte value stays exact. */
63
+ function memoryScale(spec) {
64
+ return spec.memoryGib / BASELINE_MEMORY_GIB;
65
+ }
66
+ /**
67
+ * Derive the server-level pool/cache/concurrency configuration from the
68
+ * instance type. Replaces the hand-tuned m7g.medium constants that silently
69
+ * stopped fitting when the instance type moved (the 2026-08-03 r8g.medium
70
+ * flip scaled only the container memory limit; every pool and cache stayed
71
+ * at 4 GiB values until the 2026-08-12 prod investigation caught it).
72
+ */
73
+ export function deriveClickHouseServerTuning(instanceType) {
74
+ const spec = resolveSpec(instanceType);
75
+ const scale = memoryScale(spec);
76
+ // Merge/mutation workers burn CPU — scale with vCPUs, floor at the proven
77
+ // 1-vCPU value. The ratio then allows 2x slots for merge SELECTION, while
78
+ // actual execution stays bounded by the pool.
79
+ const backgroundPoolSize = Math.max(2, 2 * spec.vcpus);
80
+ const mergesMutationsConcurrencyRatio = 2;
81
+ // ClickHouse sanity-checks the merge_tree free-entry thresholds against
82
+ // pool * ratio at startup (BAD_ARGUMENTS when a threshold exceeds the
83
+ // slot count). Deriving from the slot count keeps the invariant by
84
+ // construction: slots/4 approximates the proportion the CH defaults
85
+ // (20/8/25 against 32 slots) express, floored at 1 so small pools can
86
+ // always start a merge/mutation the moment a slot frees.
87
+ const slots = backgroundPoolSize * mergesMutationsConcurrencyRatio;
88
+ const freeEntries = Math.max(1, Math.floor(slots / 4));
89
+ return {
90
+ // Slots time-slice rather than parallelise on small hosts; 8 per vCPU is
91
+ // the proven m7g.medium value (raised 4→8 2026-07-28 for the webapp
92
+ // overview's 6-10 queries/page fan-out).
93
+ maxConcurrentQueries: BASE_MAX_CONCURRENT_QUERIES_PER_VCPU * spec.vcpus,
94
+ backgroundPoolSize,
95
+ backgroundSchedulePoolSize: Math.max(SCHEDULE_POOL_FLOOR, 4 * spec.vcpus),
96
+ // Moves are I/O-bound (S3 cold-tier uploads); fetches serve replication
97
+ // (none in this single-node design) and the broker pool serves Kafka
98
+ // (none) — both stay minimal at every size.
99
+ backgroundMovePoolSize: spec.vcpus >= 4 ? 2 : 1,
100
+ backgroundFetchesPoolSize: 1,
101
+ backgroundMessageBrokerSchedulePoolSize: 1,
102
+ mergesMutationsConcurrencyRatio,
103
+ freeEntriesToExecuteMutation: freeEntries,
104
+ freeEntriesToLowerMaxSizeOfMerge: freeEntries,
105
+ freeEntriesToExecuteOptimizeEntirePartition: freeEntries,
106
+ markCacheBytes: BASE_MARK_CACHE_BYTES * scale,
107
+ indexMarkCacheBytes: BASE_INDEX_MARK_CACHE_BYTES * scale,
108
+ queryCacheBytes: BASE_QUERY_CACHE_BYTES * scale
109
+ };
110
+ }
111
+ /**
112
+ * Derive the four workload-class default profiles from the instance type.
113
+ * Memory caps, external-spill thresholds and row-read ceilings scale with
114
+ * instance memory; thread caps scale with vCPUs. At the default
115
+ * m7g.medium this reproduces the historical hand-tuned values exactly
116
+ * (pinned by clickhouseTuning.test.ts).
117
+ *
118
+ * `high_throughput_ingest.maxConcurrentQueriesForUser` deliberately does
119
+ * NOT scale: it mirrors the webapp's client-side FIFO gate
120
+ * (`webapp/app/.server/lib/clickhouse/appQuerySlots.ts`) — a users.xml
121
+ * parity test on the webapp side enforces the pair, so the value is a
122
+ * client contract, not an instance-capacity knob.
123
+ */
124
+ export function deriveClickHouseDefaultProfiles(instanceType) {
125
+ const spec = resolveSpec(instanceType);
126
+ const scale = memoryScale(spec);
127
+ return Object.freeze({
128
+ high_throughput_ingest: Object.freeze({
129
+ maxThreads: spec.vcpus,
130
+ maxInsertThreads: spec.vcpus,
131
+ maxConcurrentQueriesForUser: 6,
132
+ logQueriesMinQueryDurationMs: 100,
133
+ optimizeMoveToPrewhere: true,
134
+ useQueryConditionCache: true,
135
+ useSkipIndexesIfFinal: true,
136
+ applyRowPolicyAfterFinal: true,
137
+ applyPrewhereAfterFinal: true,
138
+ doNotMergeAcrossPartitionsSelectFinal: true,
139
+ asyncInsert: true,
140
+ waitForAsyncInsert: true,
141
+ asyncInsertMaxDataSize: 10000000,
142
+ asyncInsertBusyTimeoutMinMs: 50,
143
+ asyncInsertBusyTimeoutMaxMs: 2000,
144
+ asyncInsertUseAdaptiveBusyTimeout: true,
145
+ asyncInsertDeduplicate: true,
146
+ inputFormatParallelParsing: false,
147
+ outputFormatParallelFormatting: false,
148
+ queryPlanOptimizeLazyMaterialization: true,
149
+ maxMemoryUsage: 1610612736 * scale, // 1.5 GiB baseline
150
+ maxMemoryUsageForUser: 2147483648 * scale, // 2 GiB baseline
151
+ maxBytesBeforeExternalSort: 536870912 * scale, // 512 MiB baseline
152
+ maxBytesBeforeExternalGroupBy: 536870912 * scale,
153
+ maxExecutionTime: 30,
154
+ // The ceiling an unbounded consumer query dies against (TOO_MANY_ROWS).
155
+ // Scales with memory so a bigger node's tables get proportionate
156
+ // headroom — but consumers must still time-bound their scans; this is
157
+ // a guard rail, not a budget to grow into.
158
+ maxRowsToRead: 10000000 * scale
159
+ }),
160
+ audit_append: Object.freeze({
161
+ maxThreads: 1,
162
+ maxInsertThreads: 1,
163
+ maxConcurrentQueriesForUser: 2,
164
+ maxMemoryUsage: 500000000 * scale,
165
+ maxExecutionTime: 10,
166
+ asyncInsert: true,
167
+ waitForAsyncInsert: true
168
+ }),
169
+ read_only: Object.freeze({
170
+ maxThreads: spec.vcpus,
171
+ maxConcurrentQueriesForUser: 1,
172
+ maxMemoryUsage: 1000000000 * scale,
173
+ maxExecutionTime: 3600
174
+ }),
175
+ ddl_admin: Object.freeze({
176
+ maxThreads: spec.vcpus,
177
+ maxConcurrentQueriesForUser: 1,
178
+ maxMemoryUsage: 1000000000 * scale,
179
+ maxExecutionTime: 1800
180
+ })
181
+ });
182
+ }
@@ -11,6 +11,13 @@ export interface BuildClickHouseUserDataOptions {
11
11
  backupBucketName: string;
12
12
  /** AWS region of the backup bucket. */
13
13
  backupBucketRegion: string;
14
+ /**
15
+ * Resolved EC2 instance type. Server pools, caches and concurrency caps
16
+ * derive from it via `deriveClickHouseServerTuning` — must be a key of
17
+ * `CLICKHOUSE_INSTANCE_SPECS` (unknown types throw at synth, same contract
18
+ * as `clickHouseTaskMemoryMiB`).
19
+ */
20
+ instanceType: string;
14
21
  /**
15
22
  * Optional S3 cold-tier configuration.
16
23
  * When omitted, single-tier hot storage on the EC2 EBS volume is used.