@fjall/components-infrastructure 10.1.2 → 11.0.0

package/dist/lib/resources/aws/database/clickhouseUserData.js

@@ -1,8 +1,10 @@
 import { CLICKHOUSE_DATA_MOUNT_PATH, CLICKHOUSE_EBS_DEVICE_NAME, CLICKHOUSE_CONFIG_SUBDIR, CLICKHOUSE_USERS_SUBDIR, CLICKHOUSE_ECS_RESERVED_MEMORY_MIB, CLICKHOUSE_HOST_METRICS, CLICKHOUSE_HTTP_PORT, CLICKHOUSE_HTTPS_PORT, CLICKHOUSE_TCP_SECURE_PORT, CLICKHOUSE_TLS_CERT_MOUNT_PATH, CLICKHOUSE_PROMETHEUS_PORT, CLICKHOUSE_UID, clickHousePasswordSha256Snippet } from "./clickhouseConstants.js";
+import { deriveClickHouseServerTuning, MAX_SERVER_MEMORY_RAM_RATIO, SYSTEM_LOG_TTL } from "./clickhouseTuning.js";
 import { renderUsersXml } from "./clickhouseXmlRenderer.js";
 export function generateServerConfigXml(options) {
     const { backupBucketName, backupBucketRegion, coldTier } = options;
     const tlsActive = options.tls !== undefined;
+    const tuning = deriveClickHouseServerTuning(options.instanceType);
     const storageBlock = coldTier !== undefined
         ? `    <storage_configuration>
         <!-- Same CH 26 rule as the no-cold-tier branch: a <local_ssd> disk
@@ -15,6 +17,15 @@ export function generateServerConfigXml(options) {
                 <endpoint>https://${coldTier.bucketName}.s3.${coldTier.region}.amazonaws.com/cold/</endpoint>
                 <use_environment_credentials>true</use_environment_credentials>
                 <metadata_path>/var/lib/clickhouse/disks/s3_cold/</metadata_path>
+                <!-- Blob garbage collection defaults to a 1 s wake — pure
+                     schedule-pool churn on this workload (prod 2026-08-12:
+                     353,035 runs removed 1 blob). Cold-tier parts move on a
+                     TTL measured in days, so a 60 s cadence loses nothing.
+                     Key: DiskObjectStorage reads
+                     <disk>.data_background_cleanup.interval_sec (CH 26.3). -->
+                <data_background_cleanup>
+                    <interval_sec>60</interval_sec>
+                </data_background_cleanup>
             </s3_cold>
         </disks>
         <policies>
@@ -89,65 +100,67 @@ export function generateServerConfigXml(options) {
          work, but explicit beats implicit and keeps the listener identical
          across network modes. -->
     <listen_host>0.0.0.0</listen_host>
-    <!-- m7g.medium tuning philosophy: 1 vCPU sustained / 4 GB / 3 GB container.
-         Single-thread CPU has no headroom (raising max_threads beyond 1 just
-         oversubscribes the kernel scheduler), so we trade RAM for CPU — size
-         caches generously so dashboard reads hit cache rather than rescanning,
-         and let a single query use most of the budget if needed.
-         Re-tune these in lockstep when DEFAULT_CLICKHOUSE_INSTANCE_TYPE moves
-         to a larger box. -->
-    <!-- 0.75 ratio gives the kernel ~750 MB cushion on the 3 GB cgroup for
-         CH's untracked allocations (libc allocator, executable image, malloc
+    <!-- Small-node tuning philosophy: trade RAM for CPU — size caches
+         generously so dashboard reads hit cache rather than rescanning, and
+         let a single query use most of the budget if needed. Every pool,
+         cache and concurrency value below is derived from the instance type
+         by deriveClickHouseServerTuning (clickhouseTuning.ts) — the
+         pre-derivation shape (hand-tuned m7g.medium constants) silently
+         stopped fitting when the instance moved to r8g.medium. -->
+    <!-- 0.75 ratio gives the kernel a cushion on the cgroup for CH's
+         untracked allocations (libc allocator, executable image, malloc
          fragmentation, jemalloc virtual reservations on ARM64). The default
-         0.9 left only ~300 MB and OOM-killed the process at boot when cache
-         + thread init pushed the cgroup over the limit (exit 137 in CW).
-         Computed at runtime from the cgroup limit so it tracks instance size
-         when DEFAULT_CLICKHOUSE_INSTANCE_TYPE moves to a larger box. -->
-    <max_server_memory_usage_to_ram_ratio>0.75</max_server_memory_usage_to_ram_ratio>
+         0.9 left only ~300 MB on the 4 GiB host and OOM-killed the process
+         at boot when cache + thread init pushed the cgroup over the limit
+         (exit 137 in CW). Computed at runtime from the cgroup limit so it
+         tracks instance size. -->
+    <max_server_memory_usage_to_ram_ratio>${MAX_SERVER_MEMORY_RAM_RATIO}</max_server_memory_usage_to_ram_ratio>
     <!-- Per-query caps (max_memory_usage, max_bytes_before_external_sort,
          max_bytes_before_external_group_by) live in the profile blocks of
          users.xml — ClickHouse 26.3.10 rejects them at top level with
          UNKNOWN_ELEMENT_IN_CONFIG. The default localhost user inherits
          ClickHouse's built-in defaults; workload users (app_writer,
          audit_writer, backup_reader, schema_admin) carry explicit caps. -->
-    <!-- 384 MB mark cache + 128 MB index mark cache. CH counts these against
-         max_server_memory_usage, so the read-cache footprint is 512 MB total
-         (~17% of the budget). On a single-tenant dashboard workload this
-         pays back fast — same queries repeat, mark hits avoid index re-scans. -->
-    <mark_cache_size>402653184</mark_cache_size>
-    <index_mark_cache_size>134217728</index_mark_cache_size>
+    <!-- Mark + index-mark caches scale linearly with instance memory
+         (384 MiB + 128 MiB at the 4 GiB baseline). CH counts these against
+         max_server_memory_usage. On a single-tenant dashboard workload they
+         pay back fast — same queries repeat, mark hits avoid index re-scans. -->
+    <mark_cache_size>${tuning.markCacheBytes}</mark_cache_size>
+    <index_mark_cache_size>${tuning.indexMarkCacheBytes}</index_mark_cache_size>
     <!-- Filesystem page cache handles uncompressed-block warmth on the
          host side; an in-CH uncompressed cache would just duplicate it. -->
     <uncompressed_cache_size>0</uncompressed_cache_size>
-    <!-- Server-wide cap across ALL users. Raised 4→8 (2026-07-28) alongside
-         high_throughput_ingest's per-user cap (2→6): the webapp overview
-         fires 6-10 queries per page load and N app tasks share this cap, so
-         4 starved dashboard reads with TOO_MANY_SIMULTANEOUS_QUERIES (CH
-         rejects rather than queues past the cap). On the 1 vCPU m7g.medium
-         the extra slots time-slice rather than parallelise — acceptable for
-         short cache-hot dashboard reads; max_threads=1 per query keeps the
-         scheduler sane. Bump further in lockstep with instance size if
-         DEFAULT_CLICKHOUSE_INSTANCE_TYPE moves to a >=2 vCPU host. -->
-    <max_concurrent_queries>8</max_concurrent_queries>
-    <background_pool_size>2</background_pool_size>
-    <background_schedule_pool_size>2</background_schedule_pool_size>
-    <background_merges_mutations_concurrency_ratio>2</background_merges_mutations_concurrency_ratio>
-    <background_move_pool_size>1</background_move_pool_size>
-    <background_fetches_pool_size>1</background_fetches_pool_size>
-    <background_message_broker_schedule_pool_size>1</background_message_broker_schedule_pool_size>
+    <!-- Server-wide cap across ALL users, 8 per vCPU (raised 4→8 2026-07-28
+         alongside high_throughput_ingest's per-user cap 2→6: the webapp
+         overview fires 6-10 queries per page load and CH rejects rather than
+         queues past the cap). On a 1 vCPU host the slots time-slice rather
+         than parallelise — acceptable for short cache-hot dashboard reads. -->
+    <max_concurrent_queries>${tuning.maxConcurrentQueries}</max_concurrent_queries>
+    <background_pool_size>${tuning.backgroundPoolSize}</background_pool_size>
+    <!-- Floored at 8 regardless of size: CH >= 25.9 caps same-type
+         background tasks at floor(pool * 0.8) and logs a warning on every
+         deferred task start — a pool of 2 (cap 1) flooded prod with ~35k
+         warnings/hour and serialised all background scheduling
+         (2026-08-12). The cap is computed once at pool creation, so this
+         value only changes behaviour after a server restart. -->
+    <background_schedule_pool_size>${tuning.backgroundSchedulePoolSize}</background_schedule_pool_size>
+    <background_merges_mutations_concurrency_ratio>${tuning.mergesMutationsConcurrencyRatio}</background_merges_mutations_concurrency_ratio>
+    <background_move_pool_size>${tuning.backgroundMovePoolSize}</background_move_pool_size>
+    <background_fetches_pool_size>${tuning.backgroundFetchesPoolSize}</background_fetches_pool_size>
+    <background_message_broker_schedule_pool_size>${tuning.backgroundMessageBrokerSchedulePoolSize}</background_message_broker_schedule_pool_size>
     <merge_tree>
         <max_suspicious_broken_parts>5</max_suspicious_broken_parts>
         <parts_to_delay_insert>150</parts_to_delay_insert>
         <parts_to_throw_insert>300</parts_to_throw_insert>
         <!-- ClickHouse 26.3.10 sanity-checks the free-pool thresholds against
              background_pool_size * background_merges_mutations_concurrency_ratio
-             (= 2 * 2 = 4 here). The defaults (20 / 8 / 25) all exceed 4 and
-             trip BAD_ARGUMENTS at startup on a 1 vCPU m7g.medium. Lower each
-             to 1 so a mutation/merge/optimise can begin as soon as one pool
-             slot is free — appropriate for the small instance footprint. -->
-        <number_of_free_entries_in_pool_to_execute_mutation>1</number_of_free_entries_in_pool_to_execute_mutation>
-        <number_of_free_entries_in_pool_to_lower_max_size_of_merge>1</number_of_free_entries_in_pool_to_lower_max_size_of_merge>
-        <number_of_free_entries_in_pool_to_execute_optimize_entire_partition>1</number_of_free_entries_in_pool_to_execute_optimize_entire_partition>
+             and trips BAD_ARGUMENTS at startup when a threshold exceeds the
+             slot count (the CH defaults 20/8/25 all exceed a small pool).
+             Derived as max(1, slots/4) so the invariant holds by
+             construction at every instance size. -->
+        <number_of_free_entries_in_pool_to_execute_mutation>${tuning.freeEntriesToExecuteMutation}</number_of_free_entries_in_pool_to_execute_mutation>
+        <number_of_free_entries_in_pool_to_lower_max_size_of_merge>${tuning.freeEntriesToLowerMaxSizeOfMerge}</number_of_free_entries_in_pool_to_lower_max_size_of_merge>
+        <number_of_free_entries_in_pool_to_execute_optimize_entire_partition>${tuning.freeEntriesToExecuteOptimizeEntirePartition}</number_of_free_entries_in_pool_to_execute_optimize_entire_partition>
     </merge_tree>
 ${tlsActive
         ? `    <https_port>${CLICKHOUSE_HTTPS_PORT}</https_port>
@@ -170,38 +183,60 @@ ${tlsActive
         <database>system</database>
         <table>query_log</table>
         <flush_interval_milliseconds>7500</flush_interval_milliseconds>
-        <ttl>event_date + INTERVAL 14 DAY DELETE</ttl>
+        <ttl>${SYSTEM_LOG_TTL}</ttl>
     </query_log>
     <!-- All system log tables need bounded retention on 80GB EBS (mirror query_log's 14-day policy). -->
     <asynchronous_metric_log>
         <database>system</database>
         <table>asynchronous_metric_log</table>
-        <ttl>event_date + INTERVAL 14 DAY DELETE</ttl>
+        <ttl>${SYSTEM_LOG_TTL}</ttl>
     </asynchronous_metric_log>
     <metric_log>
         <database>system</database>
         <table>metric_log</table>
-        <ttl>event_date + INTERVAL 14 DAY DELETE</ttl>
+        <ttl>${SYSTEM_LOG_TTL}</ttl>
     </metric_log>
     <part_log>
         <database>system</database>
         <table>part_log</table>
-        <ttl>event_date + INTERVAL 14 DAY DELETE</ttl>
+        <ttl>${SYSTEM_LOG_TTL}</ttl>
     </part_log>
     <error_log>
         <database>system</database>
         <table>error_log</table>
-        <ttl>event_date + INTERVAL 14 DAY DELETE</ttl>
+        <ttl>${SYSTEM_LOG_TTL}</ttl>
     </error_log>
+    <!-- CH creates these three by default with NO retention — the trace_log
+         family alone reached 3.5 GiB on prod before TTLs landed (2026-08-12).
+         Adding <ttl> changes the storage definition: on first boot after this
+         change CH renames each existing table to a *_N sibling and creates a
+         fresh one with the TTL. The renamed tables keep their data and need a
+         manual DROP TABLE to reclaim the disk. -->
+    <text_log>
+        <database>system</database>
+        <table>text_log</table>
+        <ttl>${SYSTEM_LOG_TTL}</ttl>
+    </text_log>
+    <trace_log>
+        <database>system</database>
+        <table>trace_log</table>
+        <ttl>${SYSTEM_LOG_TTL}</ttl>
+    </trace_log>
+    <background_schedule_pool_log>
+        <database>system</database>
+        <table>background_schedule_pool_log</table>
+        <ttl>${SYSTEM_LOG_TTL}</ttl>
+    </background_schedule_pool_log>
 ${storageBlock}
 ${backupEndpointBlock}
-    <!-- 256 MB query cache. Single-tenant dashboard pattern means the same
-         queries repeat at fixed cadences (5/15min refresh, manual reloads);
-         a hit returns in microseconds instead of re-scanning. Sized generously
+    <!-- Query cache scales with instance memory (256 MiB at the 4 GiB
+         baseline). Single-tenant dashboard pattern means the same queries
+         repeat at fixed cadences (5/15min refresh, manual reloads); a hit
+         returns in microseconds instead of re-scanning. Sized generously
          relative to other caches because the workload pattern is uniquely
          cache-friendly. -->
     <query_cache>
-        <max_size_in_bytes>268435456</max_size_in_bytes>
+        <max_size_in_bytes>${tuning.queryCacheBytes}</max_size_in_bytes>
         <max_entries>1024</max_entries>
         <max_entry_size_in_bytes>1048576</max_entry_size_in_bytes>
     </query_cache>

package/dist/lib/resources/aws/database/clickhouseXmlRenderer.js

@@ -95,6 +95,16 @@ export function renderUsersXml(opts) {
                  box that's a merge-pool starvation event. Keep TTL changes lazy: parts
                  re-evaluate TTL on their next natural merge, no forced rewrite. -->
             <materialize_ttl_after_modify>0</materialize_ttl_after_modify>
+            <!-- Official builds sample every running query's stacks at 1 s
+                 into system.trace_log. Nothing in the Fjall offering reads
+                 those samples, and on a 1-vCPU node the sampling costs CPU
+                 and disk (trace_log was the fastest-growing system table on
+                 prod, 2026-08-12). The default profile is the server-wide
+                 base, so this disables the profiler for every user; re-enable
+                 per session (SET query_profiler_real_time_period_ns = 1000000000)
+                 when actively profiling. -->
+            <query_profiler_real_time_period_ns>0</query_profiler_real_time_period_ns>
+            <query_profiler_cpu_time_period_ns>0</query_profiler_cpu_time_period_ns>
         </default>`;
     const profileBlocks = Object.entries(opts.profiles)
         .map(([name, spec]) => renderProfileBlock(name, spec))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fjall/components-infrastructure",
-  "version": "10.1.2",
+  "version": "11.0.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/fjall-tech/fjall.git",
@@ -80,8 +80,8 @@
   },
   "dependencies": {
     "@aws-sdk/client-organizations": "^3.1098.0",
-    "@fjall/generator": "^10.1.2",
-    "@fjall/util": "^10.1.2",
+    "@fjall/generator": "^11.0.0",
+    "@fjall/util": "^11.0.0",
     "constructs": "^10.7.2"
   },
   "overrides": {