@fjall/components-infrastructure 15.0.0 → 16.0.0

package/dist/lib/patterns/aws/clickhouseDatabase.js

@@ -21,7 +21,7 @@ import { toPascalCase } from "../../utils/capitaliseString.js";
 import { resolveAlertsTopic } from "../../utils/resolveAlertsTopic.js";
 import { createClickHouseAlarms, validateClickHouseAlarmThresholds, BACKUP_HEARTBEAT_DEFAULT_WINDOW_HOURS, BACKUP_HEARTBEAT_MAX_WINDOW_HOURS, METRIC_NAMESPACE, stackScopedMetricNamespace } from "../../resources/aws/monitoring/index.js";
 import { ClickHouseSchemaAdminSchema, ManagedPasswordNameSchema, ProfileSpecSchema, PROFILE_NAME_PATTERN } from "../../resources/aws/database/clickhouseSchemas.js";
-import { deriveClickHouseDefaultProfiles, deriveClickHouseServerTuning } from "../../resources/aws/database/clickhouseTuning.js";
+import { deriveClickHouseDefaultProfiles } from "../../resources/aws/database/clickhouseTuning.js";
 import { inferAmiHardwareType } from "../../resources/aws/compute/ecsConstants.js";
 import { CLICKHOUSE_DATABASE_NAME, DEFAULT_CLICKHOUSE_INSTANCE_TYPE, CLICKHOUSE_IMAGE, CLICKHOUSE_EBS_VOLUME_SIZE_GB, CLICKHOUSE_EBS_IOPS, CLICKHOUSE_EBS_THROUGHPUT_MBPS, clickHouseTaskMemoryMiB, CLICKHOUSE_HTTP_PORT, CLICKHOUSE_HTTPS_PORT, CLICKHOUSE_NATIVE_PORT, CLICKHOUSE_TCP_SECURE_PORT, CLICKHOUSE_TLS_CERT_MOUNT_PATH, CLICKHOUSE_PROMETHEUS_PORT, CLICKHOUSE_DATA_MOUNT_PATH, CLICKHOUSE_SECRET_OPTIONS, CLICKHOUSE_SERVER_ROLE_TAG, CLICKHOUSE_HOST_METRICS, clickHouseUserSecretName, CLICKHOUSE_HEALTH_CHECK, CLICKHOUSE_STOP_TIMEOUT_SECONDS, CLICKHOUSE_EBS_DEVICE_NAME, CLICKHOUSE_CONFIG_SUBDIR, CLICKHOUSE_USERS_SUBDIR, userPasswordEnvName, OPTIMISE_FINAL_SCHEDULE, REPLACING_MERGE_TREE_TABLES, OPTIMISE_MV_TABLES, CLICKHOUSE_CLOUDMAP_SERVICE_NAME, CLICKHOUSE_SERVER_CONTAINER_NAME, CLICKHOUSE_SERVICE_NAME, OPTIMISE_TASK_MEMORY_MIB, OPTIMISE_TASK_CPU_UNITS, BACKUP_SCHEDULE, BACKUP_TASK_MEMORY_MIB, BACKUP_TASK_CPU_UNITS, BACKUP_RETENTION_DAYS } from "../../resources/aws/database/clickhouseConstants.js";
 import { TlsCertGenerator } from "../../resources/aws/utilities/tlsCertGenerator.js";
@@ -65,15 +65,35 @@ function resolveClickHouseDesiredCount(contextValue, propValue) {
  * position cannot.
  */
 const CLICKHOUSE_ERROR_LINE_PATTERN = '[w1, w2, w3, w4, w5="<Error>" || w5="<Fatal>", ...]';
+/**
+ * Failed `executeQuery:` lines per 5 minutes before the alarm fires, sustained
+ * across 2 of 3 evaluation periods.
+ *
+ * ABSOLUTE, deliberately. This default used to be `3 x maxConcurrentQueries`,
+ * which read as principled and was not: a concurrency ceiling and a failure
+ * RATE are unrelated quantities, and scaling the threshold with the host size
+ * pointed the wrong way — a larger instance serves more tenants, so a given
+ * failure rate there is worse news, not more tolerable. On the 1-vCPU host it
+ * resolved to 24, meaning a steady 4-per-minute stream of failing queries
+ * never alarmed. That is the exact shape of the 2026-08-20 incident, where
+ * rejected queries surfaced to customers as missing data while every alarm
+ * stayed green.
+ *
+ * 10 is chosen against what the client now does rather than against the host:
+ * the webapp gates reads to its share of the per-user cap and retries a
+ * concurrency rejection before giving up, so a query that reaches the server
+ * AND fails is already exceptional. The 2-of-3 datapoint rule is what keeps a
+ * deploy blip or a single bad statement from paging.
+ */
+const DEFAULT_FAILED_QUERIES_PER_5_MIN = 10;
 const CLICKHOUSE_FAILED_QUERY_PATTERN = '[w1, w2, w3, w4, w5="<Error>", w6="executeQuery:", ...]';
 /**
  * Server-log signal alarm specs for the ClickHouse service's declarative
  * `logAlarms`. Three always-on signals plus a cold-tier S3 signal when the
  * cold tier is enabled. Thresholds come from the `alarms` knob with defaults
- * documented on `ClickHouseAlarmThresholds`; the failed-queries default
- * derives from the instance's `maxConcurrentQueries` so it tracks host size.
+ * documented on `ClickHouseAlarmThresholds`.
  */
-function buildClickHouseLogAlarmSpecs(thresholds, coldTierEnabled, maxConcurrentQueries) {
+function buildClickHouseLogAlarmSpecs(thresholds, coldTierEnabled) {
     const specs = [
         {
             idStem: "ClickHouseServerErrors",
@@ -90,7 +110,7 @@ function buildClickHouseLogAlarmSpecs(thresholds, coldTierEnabled, maxConcurrent
             metricName: "ClickHouseFailedQueryCount",
             description: "ClickHouse failed-query rate exceeds threshold — check system.query_log for the failing statement and error code",
             literal: CLICKHOUSE_FAILED_QUERY_PATTERN,
-            threshold: thresholds.failedQueriesPer5Min ?? 3 * maxConcurrentQueries,
+            threshold: thresholds.failedQueriesPer5Min ?? DEFAULT_FAILED_QUERIES_PER_5_MIN,
             period: Duration.minutes(5),
             evaluationPeriods: 3,
             datapointsToAlarm: 2
@@ -441,10 +461,9 @@ export class ClickHouseDatabase extends Construct {
             (App.hasInstance() ? App.getInstance().getName() : undefined);
         const alarmThresholds = typeof props.alarms === "object" ? props.alarms : {};
         validateClickHouseAlarmThresholds(alarmThresholds);
-        const serverTuning = deriveClickHouseServerTuning(instanceType);
         const serviceLogAlarms = alarmsDisabled
             ? undefined
-            : buildClickHouseLogAlarmSpecs(alarmThresholds, coldTierEnabled, serverTuning.maxConcurrentQueries);
+            : buildClickHouseLogAlarmSpecs(alarmThresholds, coldTierEnabled);
         const ecsCompute = new EcsCompute(this, "Compute", {
             type: "ecs",
             vpc,

package/dist/lib/resources/aws/database/clickhouseConstants.d.ts

@@ -42,8 +42,26 @@ export declare const DEFAULT_CLICKHOUSE_INSTANCE_TYPE = "m7g.medium";
  *  that pulls once per launch. Upstream CH CI runs its full perf + stress
  *  matrix on the Ubuntu build; Alpine is community-tier coverage. */
 export declare const CLICKHOUSE_IMAGE = "docker.io/clickhouse/clickhouse-server:26.3.17.56";
-/** EBS volume configuration. */
-export declare const CLICKHOUSE_EBS_VOLUME_SIZE_GB = 80;
+/** EBS volume configuration.
+ *
+ *  Sized for the 30-organisation baseline the 2026-08 capacity review set,
+ *  not for today's tenant count: the 80 GiB the cluster launched on binds at
+ *  roughly 20-25 organisations once `log_events` and `application_metrics`
+ *  carry a full retention window each, which is BELOW that baseline. 400 GiB
+ *  buys the headroom to reach it without a second resize, and gp3 storage is
+ *  cheap relative to an out-of-disk ClickHouse (merges stop, inserts start
+ *  failing, and the recovery is a restore rather than a resize).
+ *
+ *  Raising this alone is not enough — `buildClickHouseUserData` must grow the
+ *  ext4 filesystem to match, because ModifyVolume resizes the block device
+ *  and nothing else. The pair moves together; see the `resize2fs` step there.
+ *  Applying an increase to a RUNNING cluster therefore needs the instance to
+ *  relaunch (user data runs at boot), and EBS permits one modification per
+ *  volume per 6 hours.
+ *
+ *  IOPS and throughput stay at the gp3 baseline: they are size-independent on
+ *  gp3, and the workload is merge- and scan-bound rather than IOPS-starved. */
+export declare const CLICKHOUSE_EBS_VOLUME_SIZE_GB = 400;
 export declare const CLICKHOUSE_EBS_IOPS = 3000;
 export declare const CLICKHOUSE_EBS_THROUGHPUT_MBPS = 125;
 /** Host memory reserved from the ClickHouse container: kernel + ECS agent +

package/dist/lib/resources/aws/database/clickhouseConstants.js

@@ -42,8 +42,26 @@ export const DEFAULT_CLICKHOUSE_INSTANCE_TYPE = "m7g.medium";
  *  that pulls once per launch. Upstream CH CI runs its full perf + stress
  *  matrix on the Ubuntu build; Alpine is community-tier coverage. */
 export const CLICKHOUSE_IMAGE = "docker.io/clickhouse/clickhouse-server:26.3.17.56";
-/** EBS volume configuration. */
-export const CLICKHOUSE_EBS_VOLUME_SIZE_GB = 80;
+/** EBS volume configuration.
+ *
+ *  Sized for the 30-organisation baseline the 2026-08 capacity review set,
+ *  not for today's tenant count: the 80 GiB the cluster launched on binds at
+ *  roughly 20-25 organisations once `log_events` and `application_metrics`
+ *  carry a full retention window each, which is BELOW that baseline. 400 GiB
+ *  buys the headroom to reach it without a second resize, and gp3 storage is
+ *  cheap relative to an out-of-disk ClickHouse (merges stop, inserts start
+ *  failing, and the recovery is a restore rather than a resize).
+ *
+ *  Raising this alone is not enough — `buildClickHouseUserData` must grow the
+ *  ext4 filesystem to match, because ModifyVolume resizes the block device
+ *  and nothing else. The pair moves together; see the `resize2fs` step there.
+ *  Applying an increase to a RUNNING cluster therefore needs the instance to
+ *  relaunch (user data runs at boot), and EBS permits one modification per
+ *  volume per 6 hours.
+ *
+ *  IOPS and throughput stay at the gp3 baseline: they are size-independent on
+ *  gp3, and the workload is merge- and scan-bound rather than IOPS-starved. */
+export const CLICKHOUSE_EBS_VOLUME_SIZE_GB = 400;
 export const CLICKHOUSE_EBS_IOPS = 3000;
 export const CLICKHOUSE_EBS_THROUGHPUT_MBPS = 125;
 /** Host memory reserved from the ClickHouse container: kernel + ECS agent +

package/dist/lib/resources/aws/database/clickhouseUserData.js

@@ -490,6 +490,19 @@ fi
 mkdir -p "$MOUNT_POINT"
 mount "$DEVICE" "$MOUNT_POINT"
 
+# Grow the filesystem to fill the volume. Raising the CDK volume size issues a
+# ModifyVolume against the block device only — ext4 keeps its old size until
+# resize2fs runs, so without this line every capacity increase is INERT and the
+# disk fills at whatever ceiling the volume was first formatted at. Online
+# (post-mount) rather than offline: growing a mounted ext4 needs no prior
+# e2fsck, growing an unmounted one does.
+#
+# Idempotent — a no-op when the filesystem already spans the device, so it is
+# safe on every boot. Non-fatal: a resize failure leaves the previous, smaller
+# filesystem working, and refusing to boot over it would take the server down
+# to fix a headroom problem. The disk alarms are what escalate from here.
+resize2fs "$DEVICE" || echo "WARN: resize2fs failed — filesystem may not span the volume" >&2
+
 # Persist across reboots
 if ! grep -q "$MOUNT_POINT" /etc/fstab; then
   echo "$DEVICE $MOUNT_POINT ext4 defaults,nofail 0 2" >> /etc/fstab

package/dist/lib/resources/aws/monitoring/clickhouseAlarms.d.ts

@@ -29,8 +29,10 @@ export interface ClickHouseAlarmThresholds {
     diskCriticalThreshold?: number;
     /** `<Error>`/`<Fatal>` server-log lines per 5 min. Default 30. */
     serverErrorsPer5Min?: number;
-    /** Failed `executeQuery:` error lines per 5 min. Default derived:
-     *  3 × `deriveClickHouseServerTuning(instanceType).maxConcurrentQueries`. */
+    /** Failed `executeQuery:` error lines per 5 min. Default 10 — absolute, not
+     *  derived from the host's concurrency ceiling; see
+     *  `DEFAULT_FAILED_QUERIES_PER_5_MIN` in `patterns/aws/clickhouseDatabase.ts`
+     *  for why the size-derived default was the wrong shape. */
     failedQueriesPer5Min?: number;
     /** Background-backpressure lines ("Temporarily pause scheduling" /
      *  "Too many parts") per 15 min. Default 50. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fjall/components-infrastructure",
-  "version": "15.0.0",
+  "version": "16.0.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/fjall-tech/fjall.git",
@@ -80,8 +80,8 @@
   },
   "dependencies": {
     "@aws-sdk/client-organizations": "^3.1098.0",
-    "@fjall/generator": "^15.0.0",
-    "@fjall/util": "^15.0.0",
+    "@fjall/generator": "^16.0.0",
+    "@fjall/util": "^16.0.0",
     "constructs": "^10.7.2"
   },
   "overrides": {