@fjall/components-infrastructure 16.0.1 → 18.0.0

package/dist/lib/resources/aws/database/clickhouseBackupScript.d.ts

@@ -0,0 +1,103 @@
+/**
+ * The backup task's shell body: take the backup, then prove it restores.
+ *
+ * `BACKUP DATABASE … TO S3(…)` returning `BACKUP_CREATED` means ClickHouse
+ * finished writing and the statement did not raise. It does NOT mean the
+ * bytes it wrote can be read back — and the gap between those two is the
+ * whole reason a backup exists. So the task restores what it just wrote,
+ * into a scratch database, and says so in its own log line.
+ *
+ * The restore is a FULL one, deliberately. `SETTINGS structure_only = 1`
+ * looks like the cheap version of this check and is not a version of it at
+ * all: on 26.3.17.56 a structure-only restore of a backup whose entire
+ * `data/` tree had been deleted returned `RESTORED` and exit 0, because it
+ * reads `metadata/` and never opens a part. A verification that passes on a
+ * backup with no data in it is worse than no verification — it converts an
+ * unrestorable backup into a green signal. The full restore fails that same
+ * case loudly (`STD_EXCEPTION`, missing `checksums.txt`), which is the
+ * behaviour the alarm is built on.
+ */
+export interface ClickHouseBackupScriptParams {
+    /** Fully-formed `clickhouse-client …` invocation, minus `--query`. */
+    readonly client: string;
+    /** Database the backup covers, and the restore compares against. */
+    readonly databaseName: string;
+    /** `https://<bucket>.s3.<region>.amazonaws.com/backup/` — trailing slash. */
+    readonly backupDestUrl: string;
+    /** Critical free-space floor in GiB; the verify gate keeps twice this
+     *  clear so the check can never be the thing that pages. */
+    readonly diskFreeCriticalGib: number;
+    /** Declared data-volume size in GiB; bounds the verify restore's timeout
+     *  via `clickHouseBackupVerifyTimeoutSeconds`. */
+    readonly storageGb: number;
+    /** When false, the script takes the backup and stops — no restore verify.
+     *  The caller (`ClickHouseDatabase`) also withholds the verify alarms, so
+     *  the two halves of the contract disable together. */
+    readonly verify: boolean;
+}
+/**
+ * Builds the backup + restore-verify script.
+ *
+ * Statement order carries decisions that are not obvious from reading it
+ * forward:
+ *
+ * 1. **The scratch database is dropped FIRST, not only last.** A restore
+ *    onto an existing database of that name succeeds silently — verified on
+ *    26.3.17.56 — so a run that died between RESTORE and DROP would leave
+ *    tables behind that satisfy the next run's table-set comparison no
+ *    matter what the new backup contains. Dropping on the way in also makes
+ *    the free-space reading below honest, since a leftover scratch copy is
+ *    occupying the disk being measured.
+ *
+ * 2. **The live row count is sampled BEFORE the backup, and it arms the
+ *    zero-rows check.** A restore of a genuinely empty database restores
+ *    zero rows and is CORRECT; failing it would page FAILED on every fresh
+ *    stack from night one. So `restored_rows=0` is a failure only when the
+ *    source had rows at backup time. A failed source sample reads as
+ *    `unknown`, which arms the check — an unreadable source is not evidence
+ *    the database was empty (fail closed, per the read-failure posture).
+ *
+ * 3. **The restore is gated on free space, and the gate reads the same
+ *    quantity the critical disk alarm guards.** A full restore writes a
+ *    second copy of the database onto the volume it is verifying. That is
+ *    affordable at today's size and stops being affordable at some larger
+ *    one, and the failure mode if it is left ungated is the worst kind: the
+ *    backup check fills the disk that the backups exist to protect. When
+ *    the copy would not leave `diskFreeCriticalGib × margin` clear, the run
+ *    skips the restore and SAYS SO with its own marker. Silent degradation
+ *    is not available here — a skipped verify that logged nothing would be
+ *    indistinguishable from a passing one. The same discipline covers the
+ *    probe itself: a failed free-space read emits the SKIPPED marker (the
+ *    verify did not run, which is exactly what that marker means) and exits
+ *    non-zero, rather than being the one silent path out of the script.
+ *
+ * 4. **The restore is bounded by `timeout`, sized from the volume.** A hung
+ *    restore is the one backup failure with no signal: `BACKUP_CREATED` is
+ *    already emitted so the heartbeat is satisfied, and the verify alarms
+ *    count markers rather than their absence. The bound converts that
+ *    silence into exit 124, which takes the same branch as a raised
+ *    restore. It scales with `storageGb` because the largest restore the
+ *    free-space gate admits scales with the volume — a flat bound would
+ *    misreport every healthy verify as FAILED past the size where the
+ *    restore outgrows it.
+ *
+ * 5. **The comparison is on the table SET, never on row counts — and it
+ *    ignores materialised-view inner tables.** The database keeps ingesting
+ *    during and after the backup, so the restored copy is a point-in-time
+ *    from somewhere inside the backup window and can never be expected to
+ *    match live counts. Rows are checked only against zero (armed per 2).
+ *    MV inner tables (`.inner_id.<uuid>` on Atomic databases) are excluded
+ *    from both sides of the set comparison: `RESTORE … AS <scratch>` on the
+ *    same server assigns NEW UUIDs, so the live database's inner names can
+ *    never appear in the scratch database and a raw name comparison reports
+ *    every implicit MV as missing — a FAILED page against a good backup.
+ *    The MVs themselves (and their data, via the restored inner tables)
+ *    remain fully compared.
+ *
+ * Every statement carries an explicit `|| …` failure branch rather than
+ * relying on `set -e`. The TLS preamble this script is concatenated onto
+ * sets `set -eu`, but ONLY when TLS is active — so a script depending on
+ * `set -e` would abort on the first failure with TLS on and blunder past it
+ * with TLS off. Explicit branches behave identically either way.
+ */
+export declare function buildClickHouseBackupScript(params: ClickHouseBackupScriptParams): string;

package/dist/lib/resources/aws/database/clickhouseBackupScript.js

@@ -0,0 +1,123 @@
+/**
+ * The backup task's shell body: take the backup, then prove it restores.
+ *
+ * `BACKUP DATABASE … TO S3(…)` returning `BACKUP_CREATED` means ClickHouse
+ * finished writing and the statement did not raise. It does NOT mean the
+ * bytes it wrote can be read back — and the gap between those two is the
+ * whole reason a backup exists. So the task restores what it just wrote,
+ * into a scratch database, and says so in its own log line.
+ *
+ * The restore is a FULL one, deliberately. `SETTINGS structure_only = 1`
+ * looks like the cheap version of this check and is not a version of it at
+ * all: on 26.3.17.56 a structure-only restore of a backup whose entire
+ * `data/` tree had been deleted returned `RESTORED` and exit 0, because it
+ * reads `metadata/` and never opens a part. A verification that passes on a
+ * backup with no data in it is worse than no verification — it converts an
+ * unrestorable backup into a green signal. The full restore fails that same
+ * case loudly (`STD_EXCEPTION`, missing `checksums.txt`), which is the
+ * behaviour the alarm is built on.
+ */
+import { CLICKHOUSE_BACKUP_SCRATCH_DATABASE, CLICKHOUSE_BACKUP_VERIFY_FREE_SPACE_MARGIN, CLICKHOUSE_BACKUP_VERIFY_FAILED_MARKER, CLICKHOUSE_BACKUP_VERIFY_OK_MARKER, CLICKHOUSE_BACKUP_VERIFY_SKIPPED_MARKER, clickHouseBackupVerifyTimeoutSeconds } from "./clickhouseConstants.js";
+const BYTES_PER_GIB = 1024 ** 3;
+/**
+ * Builds the backup + restore-verify script.
+ *
+ * Statement order carries decisions that are not obvious from reading it
+ * forward:
+ *
+ * 1. **The scratch database is dropped FIRST, not only last.** A restore
+ *    onto an existing database of that name succeeds silently — verified on
+ *    26.3.17.56 — so a run that died between RESTORE and DROP would leave
+ *    tables behind that satisfy the next run's table-set comparison no
+ *    matter what the new backup contains. Dropping on the way in also makes
+ *    the free-space reading below honest, since a leftover scratch copy is
+ *    occupying the disk being measured.
+ *
+ * 2. **The live row count is sampled BEFORE the backup, and it arms the
+ *    zero-rows check.** A restore of a genuinely empty database restores
+ *    zero rows and is CORRECT; failing it would page FAILED on every fresh
+ *    stack from night one. So `restored_rows=0` is a failure only when the
+ *    source had rows at backup time. A failed source sample reads as
+ *    `unknown`, which arms the check — an unreadable source is not evidence
+ *    the database was empty (fail closed, per the read-failure posture).
+ *
+ * 3. **The restore is gated on free space, and the gate reads the same
+ *    quantity the critical disk alarm guards.** A full restore writes a
+ *    second copy of the database onto the volume it is verifying. That is
+ *    affordable at today's size and stops being affordable at some larger
+ *    one, and the failure mode if it is left ungated is the worst kind: the
+ *    backup check fills the disk that the backups exist to protect. When
+ *    the copy would not leave `diskFreeCriticalGib × margin` clear, the run
+ *    skips the restore and SAYS SO with its own marker. Silent degradation
+ *    is not available here — a skipped verify that logged nothing would be
+ *    indistinguishable from a passing one. The same discipline covers the
+ *    probe itself: a failed free-space read emits the SKIPPED marker (the
+ *    verify did not run, which is exactly what that marker means) and exits
+ *    non-zero, rather than being the one silent path out of the script.
+ *
+ * 4. **The restore is bounded by `timeout`, sized from the volume.** A hung
+ *    restore is the one backup failure with no signal: `BACKUP_CREATED` is
+ *    already emitted so the heartbeat is satisfied, and the verify alarms
+ *    count markers rather than their absence. The bound converts that
+ *    silence into exit 124, which takes the same branch as a raised
+ *    restore. It scales with `storageGb` because the largest restore the
+ *    free-space gate admits scales with the volume — a flat bound would
+ *    misreport every healthy verify as FAILED past the size where the
+ *    restore outgrows it.
+ *
+ * 5. **The comparison is on the table SET, never on row counts — and it
+ *    ignores materialised-view inner tables.** The database keeps ingesting
+ *    during and after the backup, so the restored copy is a point-in-time
+ *    from somewhere inside the backup window and can never be expected to
+ *    match live counts. Rows are checked only against zero (armed per 2).
+ *    MV inner tables (`.inner_id.<uuid>` on Atomic databases) are excluded
+ *    from both sides of the set comparison: `RESTORE … AS <scratch>` on the
+ *    same server assigns NEW UUIDs, so the live database's inner names can
+ *    never appear in the scratch database and a raw name comparison reports
+ *    every implicit MV as missing — a FAILED page against a good backup.
+ *    The MVs themselves (and their data, via the restored inner tables)
+ *    remain fully compared.
+ *
+ * Every statement carries an explicit `|| …` failure branch rather than
+ * relying on `set -e`. The TLS preamble this script is concatenated onto
+ * sets `set -eu`, but ONLY when TLS is active — so a script depending on
+ * `set -e` would abort on the first failure with TLS on and blunder past it
+ * with TLS off. Explicit branches behave identically either way.
+ */
+export function buildClickHouseBackupScript(params) {
+    const { client, databaseName, backupDestUrl, diskFreeCriticalGib, storageGb, verify } = params;
+    const backupStatements = [
+        "STAMP=$(date +%Y%m%d-%H%M%S)",
+        `DEST="${backupDestUrl}backup-$STAMP/"`,
+        `${client} --query "BACKUP DATABASE ${databaseName} TO S3('$DEST')" || exit 1`
+    ];
+    if (!verify) {
+        return backupStatements.join("; ");
+    }
+    const scratch = CLICKHOUSE_BACKUP_SCRATCH_DATABASE;
+    const requiredFreeBytes = Math.round(diskFreeCriticalGib *
+        BYTES_PER_GIB *
+        CLICKHOUSE_BACKUP_VERIFY_FREE_SPACE_MARGIN);
+    const verifyTimeoutSeconds = clickHouseBackupVerifyTimeoutSeconds(storageGb);
+    const dropScratch = `${client} --query "DROP DATABASE IF EXISTS ${scratch} SYNC"`;
+    const sourceRowsQuery = `SELECT sum(rows) FROM system.parts WHERE database = '${databaseName}' AND active`;
+    const fitVerdictQuery = `SELECT if((SELECT free_space FROM system.disks WHERE name = 'default')` +
+        ` - (SELECT sum(bytes_on_disk) FROM system.parts WHERE database = '${databaseName}' AND active)` +
+        ` >= ${requiredFreeBytes}, 'FITS', 'SKIP')`;
+    const missingTablesQuery = `SELECT count() FROM (SELECT name FROM system.tables WHERE database = '${databaseName}' AND name NOT LIKE '.inner%'` +
+        ` EXCEPT SELECT name FROM system.tables WHERE database = '${scratch}' AND name NOT LIKE '.inner%')`;
+    const restoredRowsQuery = `SELECT sum(rows) FROM system.parts WHERE database = '${scratch}' AND active`;
+    return [
+        `${dropScratch} || exit 1`,
+        `SRCROWS=$(${client} --query "${sourceRowsQuery}") || SRCROWS=unknown`,
+        ...backupStatements,
+        `FIT=$(${client} --query "${fitVerdictQuery}") || { echo "${CLICKHOUSE_BACKUP_VERIFY_SKIPPED_MARKER} $DEST free-space probe failed — verify did not run"; exit 1; }`,
+        `if [ "$FIT" != "FITS" ]; then echo "${CLICKHOUSE_BACKUP_VERIFY_SKIPPED_MARKER} $DEST restore would not leave ${requiredFreeBytes} bytes free"; exit 0; fi`,
+        `timeout ${verifyTimeoutSeconds} ${client} --query "RESTORE DATABASE ${databaseName} AS ${scratch} FROM S3('$DEST')" || { echo "${CLICKHOUSE_BACKUP_VERIFY_FAILED_MARKER} $DEST restore raised or exceeded ${verifyTimeoutSeconds}s"; ${dropScratch} || true; exit 1; }`,
+        `MISSING=$(${client} --query "${missingTablesQuery}") || MISSING=unknown`,
+        `ROWS=$(${client} --query "${restoredRowsQuery}") || ROWS=0`,
+        `${dropScratch} || true`,
+        `if [ "$MISSING" != "0" ] || { [ "$ROWS" = "0" ] && [ "$SRCROWS" != "0" ]; }; then echo "${CLICKHOUSE_BACKUP_VERIFY_FAILED_MARKER} $DEST missing_tables=$MISSING restored_rows=$ROWS source_rows=$SRCROWS"; exit 1; fi`,
+        `echo "${CLICKHOUSE_BACKUP_VERIFY_OK_MARKER} $DEST restored_rows=$ROWS source_rows=$SRCROWS"`
+    ].join("; ");
+}

package/dist/lib/resources/aws/database/clickhouseConstants.d.ts

@@ -1,3 +1,4 @@
+import type { ClickHouseInstanceType } from "@fjall/util/clickhouse";
 /** Database name created at ClickHouse bootstrap; consumed by BACKUP DATABASE,
  *  OPTIMIZE TABLE, and the DatabaseName CfnOutput so all four sites share one source. */
 export declare const CLICKHOUSE_DATABASE_NAME = "analytics";
@@ -42,28 +43,67 @@ export declare const DEFAULT_CLICKHOUSE_INSTANCE_TYPE = "m7g.medium";
  *  that pulls once per launch. Upstream CH CI runs its full perf + stress
  *  matrix on the Ubuntu build; Alpine is community-tier coverage. */
 export declare const CLICKHOUSE_IMAGE = "docker.io/clickhouse/clickhouse-server:26.3.17.56";
+/** Absolute floor of the disk-critical alarm, in GiB.
+ *
+ *  A pure percentage cannot express "about to break": 85 % used is 12 GiB
+ *  free on an 80 GiB volume and 600 GiB free on a 4 TiB one, and only one of
+ *  those is an incident. But a pure constant cannot either, because the other
+ *  half of the failure scales with the data — ClickHouse will not select a
+ *  merge whose source parts need more than the free space allows, and parts
+ *  grow with the table. So the default floor is the LARGER of this constant
+ *  and `CLICKHOUSE_DISK_FREE_CRITICAL_SHARE` of the volume: the constant
+ *  guards response time on a small disk, the share guards merge headroom on a
+ *  large one. See `resolveClickHouseStorage`.
+ *
+ *  20 GiB is days of ingest headroom at the observed rate — enough lead time
+ *  to grow the volume, which takes an instance restart and cannot be done in
+ *  minutes. */
+export declare const CLICKHOUSE_DISK_FREE_CRITICAL_GIB = 20;
+/** Share of the volume the disk-critical floor rises to on larger disks.
+ *  At 5 %, a 4 TiB volume pages with 200 GiB free — roughly twice a large
+ *  part, so merges still have room to complete while the operator responds.
+ *  The two halves cross over at 400 GiB, the default volume size. */
+export declare const CLICKHOUSE_DISK_FREE_CRITICAL_SHARE = 0.05;
 /** EBS volume configuration.
  *
- *  Sized for the 30-organisation baseline the 2026-08 capacity review set,
- *  not for today's tenant count: the 80 GiB the cluster launched on binds at
- *  roughly 20-25 organisations once `log_events` and `application_metrics`
- *  carry a full retention window each, which is BELOW that baseline. 400 GiB
- *  buys the headroom to reach it without a second resize, and gp3 storage is
- *  cheap relative to an out-of-disk ClickHouse (merges stop, inserts start
- *  failing, and the recovery is a restore rather than a resize).
- *
- *  Raising this alone is not enough — `buildClickHouseUserData` must grow the
- *  ext4 filesystem to match, because ModifyVolume resizes the block device
- *  and nothing else. The pair moves together; see the `resize2fs` step there.
- *  Applying an increase to a RUNNING cluster therefore needs the instance to
- *  relaunch (user data runs at boot), and EBS permits one modification per
- *  volume per 6 hours.
- *
- *  IOPS and throughput stay at the gp3 baseline: they are size-independent on
- *  gp3, and the workload is merge- and scan-bound rather than IOPS-starved. */
+ *  `CLICKHOUSE_EBS_VOLUME_SIZE_GB` is the DEFAULT for
+ *  `ClickHouseDatabaseProps.storageGb`, not a fixed size. 400 GiB is sized
+ *  for a mid-size multi-tenant analytics workload — order of tens of tenant
+ *  organisations each carrying full `log_events` / `application_metrics`
+ *  retention windows — with enough headroom that the first resize is a
+ *  deliberate capacity decision rather than an early surprise. (The
+ *  reference deployment's history is the evidence: 80 GiB bound at roughly
+ *  two-thirds of that scale.) gp3 storage is cheap relative to an
+ *  out-of-disk ClickHouse — merges stop, inserts start failing, and the
+ *  recovery is a restore rather than a resize. Workloads outside that
+ *  profile set `storageGb` directly, exactly as they would size an RDS
+ *  volume.
+ *
+ *  Raising the size alone is not enough — `buildClickHouseUserData` must grow
+ *  the ext4 filesystem to match, because ModifyVolume resizes the block
+ *  device and nothing else. The pair moves together: the declared size is
+ *  embedded in the user data, so a `storageGb` change versions the launch
+ *  template and triggers the instance refresh whose boot runs `resize2fs`.
+ *  EBS permits one modification per volume per 6 hours.
+ *
+ *  IOPS and throughput default to the gp3 baseline — the figures the volume
+ *  price already includes — and are the construct's `iops` / `throughputMbps`
+ *  props, the way RDS exposes provisioned IOPS and throughput on gp3. They are
+ *  size-independent on gp3 and the default workload is merge- and scan-bound
+ *  rather than IOPS-starved, so the baseline is the right default; a caller
+ *  whose scans are bandwidth-bound raises them, within the bounds and ratios
+ *  `resolveClickHouseVolumePerformance` enforces. */
 export declare const CLICKHOUSE_EBS_VOLUME_SIZE_GB = 400;
 export declare const CLICKHOUSE_EBS_IOPS = 3000;
 export declare const CLICKHOUSE_EBS_THROUGHPUT_MBPS = 125;
+/** Volume and backup-retention bounds. Re-exported, NOT redeclared:
+ *  `@fjall/generator` enforces the same numbers when it scaffolds or rewrites
+ *  an `infrastructure.ts` declaration, and a second copy here is exactly the
+ *  drift `backupRetentionDays` shipped for as long as it borrowed the RDS
+ *  schema (generator 35 vs construct 3650). One source, two enforcement
+ *  sites. */
+export { CLICKHOUSE_MIN_STORAGE_GB, CLICKHOUSE_MAX_STORAGE_GB, CLICKHOUSE_MIN_IOPS, CLICKHOUSE_MAX_IOPS, CLICKHOUSE_MAX_IOPS_PER_GB, CLICKHOUSE_MIN_THROUGHPUT_MBPS, CLICKHOUSE_MAX_THROUGHPUT_MBPS, CLICKHOUSE_IOPS_PER_THROUGHPUT_MBPS, CLICKHOUSE_MIN_BACKUP_RETENTION_DAYS, CLICKHOUSE_MAX_BACKUP_RETENTION_DAYS, CLICKHOUSE_INSTANCE_TYPES } from "@fjall/util/clickhouse";
+export type { ClickHouseInstanceType } from "@fjall/util/clickhouse";
 /** Host memory reserved from the ClickHouse container: kernel + ECS agent +
  *  the host-metrics timer. The ECS agent advertises MemTotal minus
  *  CLICKHOUSE_ECS_RESERVED_MEMORY_MIB, and MemTotal itself runs below the
@@ -81,13 +121,25 @@ export declare const CLICKHOUSE_ECS_RESERVED_MEMORY_MIB = 256;
 /** Hardware spec of every instance type the ClickHouse construct supports.
  *  Values are the AWS nominal figures. Single source of truth for BOTH the
  *  container memory limit (`clickHouseTaskMemoryMiB`) and the derived server
- *  tuning + default profiles (`clickhouseTuning.ts`) — adding a type here is
- *  the only step needed to support it end to end. */
+ *  tuning + default profiles (`clickhouseTuning.ts`).
+ *
+ *  Keyed on `ClickHouseInstanceType` — the supported-type vocabulary lives in
+ *  `@fjall/util/clickhouse` so the generator can validate `instanceType` at
+ *  scaffold time against the same list. The `Record` key type makes the
+ *  parity two-way at compile time: a vocabulary entry without a spec row is a
+ *  missing-property error, a spec row outside the vocabulary is an
+ *  excess-property error. Supporting a new type = add it to the tuple there
+ *  and the spec row here; everything else derives. */
 export interface ClickHouseInstanceSpec {
     vcpus: number;
     memoryGib: number;
 }
-export declare const CLICKHOUSE_INSTANCE_SPECS: Record<string, ClickHouseInstanceSpec>;
+export declare const CLICKHOUSE_INSTANCE_SPECS: Record<ClickHouseInstanceType, ClickHouseInstanceSpec>;
+/** Guarded string-keyed lookup into `CLICKHOUSE_INSTANCE_SPECS` for
+ *  caller-supplied types (props carry `string`); `undefined` for unknown
+ *  types, which every caller converts into its own named synth throw. The
+ *  exhaustively-keyed Record above stays the compile-time contract. */
+export declare function clickHouseInstanceSpec(instanceType: string): ClickHouseInstanceSpec | undefined;
 /** Derived memory-only view of CLICKHOUSE_INSTANCE_SPECS. */
 export declare const CLICKHOUSE_INSTANCE_MEMORY_GIB: Record<string, number>;
 /** ECS container memory for the ClickHouse server task, derived from the
@@ -179,6 +231,7 @@ export declare const CLICKHOUSE_HOST_METRICS: {
     readonly namespace: "CWAgent";
     readonly memoryMetric: "mem_used_percent";
     readonly diskMetric: "disk_used_percent";
+    readonly diskFreeMetric: "disk_free_gib";
     readonly asgDimension: "AutoScalingGroupName";
 };
 /** Shared secret generation options (all ClickHouse users share the same policy). */
@@ -248,3 +301,62 @@ export declare const BACKUP_TASK_MEMORY_MIB = 256;
 export declare const BACKUP_TASK_CPU_UNITS = 256;
 /** Backup object expiration: 14 days (retains 14 daily snapshots). */
 export declare const BACKUP_RETENTION_DAYS = 14;
+/** Scratch database the backup task restores into to prove the backup it just
+ *  wrote can be read back. Dropped on the way in and on the way out; never
+ *  queried by the application. */
+export declare const CLICKHOUSE_BACKUP_SCRATCH_DATABASE = "fjall_backup_verify";
+/** Multiple of the critical free-space floor the verify restore must leave
+ *  clear to proceed.
+ *
+ *  A full restore writes a second copy of the database onto the volume it is
+ *  verifying, so the check has to be bounded by the same quantity the disk
+ *  alarm pages on — otherwise the mechanism that protects the data is also
+ *  the mechanism most likely to fill the disk. Twice the floor rather than
+ *  once so a verify can never itself be the thing that pages: at exactly the
+ *  floor the restore would land the volume on the alarm threshold and hold it
+ *  there until the scratch database is dropped. */
+export declare const CLICKHOUSE_BACKUP_VERIFY_FREE_SPACE_MARGIN = 2;
+/** Backup task log markers. Distinct full tokens, because the CloudWatch
+ *  metric filters that read them match on substrings — a shared `BACKUP_`
+ *  stem would make the OK filter count the failures too. */
+export declare const CLICKHOUSE_BACKUP_VERIFY_OK_MARKER = "BACKUP_VERIFY_OK";
+export declare const CLICKHOUSE_BACKUP_VERIFY_FAILED_MARKER = "BACKUP_VERIFY_FAILED";
+export declare const CLICKHOUSE_BACKUP_VERIFY_SKIPPED_MARKER = "BACKUP_VERIFY_SKIPPED_FOR_SIZE";
+/** Bounds on the verify restore's wall-clock ceiling, in seconds.
+ *
+ *  Without a ceiling a hung RESTORE is the one backup failure with NO signal
+ *  at all: BACKUP_CREATED has already been emitted so the heartbeat is
+ *  satisfied, and the verify alarms count markers rather than their absence,
+ *  so a task that never finishes reports nothing. `timeout` turns that
+ *  silence into exit 124, which takes the same branch as a raised restore
+ *  and pages.
+ *
+ *  The ceiling scales with the volume (`clickHouseBackupVerifyTimeoutSeconds`
+ *  below) because the largest restore the free-space gate admits scales with
+ *  it: on the default 400 GiB the floor of one hour already covers it, but a
+ *  multi-TiB volume can legitimately need hours, and a fixed hour would page
+ *  FAILED on every healthy verify past the size where the restore outgrows
+ *  it. The 20-hour cap keeps a daily-scheduled verify from overlapping its
+ *  successor — two concurrent runs share the scratch database destructively,
+ *  so the bound must land inside the schedule interval. */
+export declare const CLICKHOUSE_BACKUP_VERIFY_MIN_TIMEOUT_SECONDS = 3600;
+export declare const CLICKHOUSE_BACKUP_VERIFY_MAX_TIMEOUT_SECONDS = 72000;
+/** Verify-restore timeout for a given data-volume size.
+ *
+ *  Worst case admitted by the free-space gate is a database of roughly half
+ *  the volume (it must fit beside itself). Writing that back at the gp3
+ *  baseline throughput (`CLICKHOUSE_EBS_THROUGHPUT_MBPS`) bounds the healthy
+ *  duration; doubled for read + merge overhead, then clamped to the
+ *  floor/cap pair above. At the 400 GiB default this resolves to the
+ *  one-hour floor, so the default behaviour is unchanged from the flat bound
+ *  it replaced.
+ *
+ *  Deliberately the BASELINE throughput, not the provisioned `throughputMbps`
+ *  prop. The volume's provisioned figure is an upper bound the instance may
+ *  not reach — an m7g.medium's EBS bandwidth sits under the gp3 baseline, and
+ *  no default-tier host reaches a provisioned 1,000 MiB/s — so a bound that
+ *  tightened with provisioning would page FAILED on healthy restores on
+ *  exactly the hosts customers start on. The baseline-derived bound is loose
+ *  on a provisioned volume, which only delays a hung-restore page; the cap
+ *  keeps that delay inside the schedule interval either way. */
+export declare function clickHouseBackupVerifyTimeoutSeconds(storageGb: number): number;

package/dist/lib/resources/aws/database/clickhouseConstants.js

@@ -42,28 +42,66 @@ export const DEFAULT_CLICKHOUSE_INSTANCE_TYPE = "m7g.medium";
  *  that pulls once per launch. Upstream CH CI runs its full perf + stress
  *  matrix on the Ubuntu build; Alpine is community-tier coverage. */
 export const CLICKHOUSE_IMAGE = "docker.io/clickhouse/clickhouse-server:26.3.17.56";
+/** Absolute floor of the disk-critical alarm, in GiB.
+ *
+ *  A pure percentage cannot express "about to break": 85 % used is 12 GiB
+ *  free on an 80 GiB volume and 600 GiB free on a 4 TiB one, and only one of
+ *  those is an incident. But a pure constant cannot either, because the other
+ *  half of the failure scales with the data — ClickHouse will not select a
+ *  merge whose source parts need more than the free space allows, and parts
+ *  grow with the table. So the default floor is the LARGER of this constant
+ *  and `CLICKHOUSE_DISK_FREE_CRITICAL_SHARE` of the volume: the constant
+ *  guards response time on a small disk, the share guards merge headroom on a
+ *  large one. See `resolveClickHouseStorage`.
+ *
+ *  20 GiB is days of ingest headroom at the observed rate — enough lead time
+ *  to grow the volume, which takes an instance restart and cannot be done in
+ *  minutes. */
+export const CLICKHOUSE_DISK_FREE_CRITICAL_GIB = 20;
+/** Share of the volume the disk-critical floor rises to on larger disks.
+ *  At 5 %, a 4 TiB volume pages with 200 GiB free — roughly twice a large
+ *  part, so merges still have room to complete while the operator responds.
+ *  The two halves cross over at 400 GiB, the default volume size. */
+export const CLICKHOUSE_DISK_FREE_CRITICAL_SHARE = 0.05;
 /** EBS volume configuration.
  *
- *  Sized for the 30-organisation baseline the 2026-08 capacity review set,
- *  not for today's tenant count: the 80 GiB the cluster launched on binds at
- *  roughly 20-25 organisations once `log_events` and `application_metrics`
- *  carry a full retention window each, which is BELOW that baseline. 400 GiB
- *  buys the headroom to reach it without a second resize, and gp3 storage is
- *  cheap relative to an out-of-disk ClickHouse (merges stop, inserts start
- *  failing, and the recovery is a restore rather than a resize).
+ *  `CLICKHOUSE_EBS_VOLUME_SIZE_GB` is the DEFAULT for
+ *  `ClickHouseDatabaseProps.storageGb`, not a fixed size. 400 GiB is sized
+ *  for a mid-size multi-tenant analytics workload — order of tens of tenant
+ *  organisations each carrying full `log_events` / `application_metrics`
+ *  retention windows — with enough headroom that the first resize is a
+ *  deliberate capacity decision rather than an early surprise. (The
+ *  reference deployment's history is the evidence: 80 GiB bound at roughly
+ *  two-thirds of that scale.) gp3 storage is cheap relative to an
+ *  out-of-disk ClickHouse — merges stop, inserts start failing, and the
+ *  recovery is a restore rather than a resize. Workloads outside that
+ *  profile set `storageGb` directly, exactly as they would size an RDS
+ *  volume.
  *
- *  Raising this alone is not enough — `buildClickHouseUserData` must grow the
- *  ext4 filesystem to match, because ModifyVolume resizes the block device
- *  and nothing else. The pair moves together; see the `resize2fs` step there.
- *  Applying an increase to a RUNNING cluster therefore needs the instance to
- *  relaunch (user data runs at boot), and EBS permits one modification per
- *  volume per 6 hours.
+ *  Raising the size alone is not enough — `buildClickHouseUserData` must grow
+ *  the ext4 filesystem to match, because ModifyVolume resizes the block
+ *  device and nothing else. The pair moves together: the declared size is
+ *  embedded in the user data, so a `storageGb` change versions the launch
+ *  template and triggers the instance refresh whose boot runs `resize2fs`.
+ *  EBS permits one modification per volume per 6 hours.
  *
- *  IOPS and throughput stay at the gp3 baseline: they are size-independent on
- *  gp3, and the workload is merge- and scan-bound rather than IOPS-starved. */
+ *  IOPS and throughput default to the gp3 baseline — the figures the volume
+ *  price already includes — and are the construct's `iops` / `throughputMbps`
+ *  props, the way RDS exposes provisioned IOPS and throughput on gp3. They are
+ *  size-independent on gp3 and the default workload is merge- and scan-bound
+ *  rather than IOPS-starved, so the baseline is the right default; a caller
+ *  whose scans are bandwidth-bound raises them, within the bounds and ratios
+ *  `resolveClickHouseVolumePerformance` enforces. */
 export const CLICKHOUSE_EBS_VOLUME_SIZE_GB = 400;
 export const CLICKHOUSE_EBS_IOPS = 3000;
 export const CLICKHOUSE_EBS_THROUGHPUT_MBPS = 125;
+/** Volume and backup-retention bounds. Re-exported, NOT redeclared:
+ *  `@fjall/generator` enforces the same numbers when it scaffolds or rewrites
+ *  an `infrastructure.ts` declaration, and a second copy here is exactly the
+ *  drift `backupRetentionDays` shipped for as long as it borrowed the RDS
+ *  schema (generator 35 vs construct 3650). One source, two enforcement
+ *  sites. */
+export { CLICKHOUSE_MIN_STORAGE_GB, CLICKHOUSE_MAX_STORAGE_GB, CLICKHOUSE_MIN_IOPS, CLICKHOUSE_MAX_IOPS, CLICKHOUSE_MAX_IOPS_PER_GB, CLICKHOUSE_MIN_THROUGHPUT_MBPS, CLICKHOUSE_MAX_THROUGHPUT_MBPS, CLICKHOUSE_IOPS_PER_THROUGHPUT_MBPS, CLICKHOUSE_MIN_BACKUP_RETENTION_DAYS, CLICKHOUSE_MAX_BACKUP_RETENTION_DAYS, CLICKHOUSE_INSTANCE_TYPES } from "@fjall/util/clickhouse";
 /** Host memory reserved from the ClickHouse container: kernel + ECS agent +
  *  the host-metrics timer. The ECS agent advertises MemTotal minus
  *  CLICKHOUSE_ECS_RESERVED_MEMORY_MIB, and MemTotal itself runs below the
@@ -93,6 +131,15 @@ export const CLICKHOUSE_INSTANCE_SPECS = {
     "r7g.xlarge": { vcpus: 4, memoryGib: 32 },
     "r8g.xlarge": { vcpus: 4, memoryGib: 32 }
 };
+/** Guarded string-keyed lookup into `CLICKHOUSE_INSTANCE_SPECS` for
+ *  caller-supplied types (props carry `string`); `undefined` for unknown
+ *  types, which every caller converts into its own named synth throw. The
+ *  exhaustively-keyed Record above stays the compile-time contract. */
+export function clickHouseInstanceSpec(instanceType) {
+    return Object.hasOwn(CLICKHOUSE_INSTANCE_SPECS, instanceType)
+        ? CLICKHOUSE_INSTANCE_SPECS[instanceType]
+        : undefined;
+}
 /** Derived memory-only view of CLICKHOUSE_INSTANCE_SPECS. */
 export const CLICKHOUSE_INSTANCE_MEMORY_GIB = Object.fromEntries(Object.entries(CLICKHOUSE_INSTANCE_SPECS).map(([type, spec]) => [
     type,
@@ -210,6 +257,7 @@ export const CLICKHOUSE_HOST_METRICS = {
     namespace: "CWAgent",
     memoryMetric: "mem_used_percent",
     diskMetric: "disk_used_percent",
+    diskFreeMetric: "disk_free_gib",
     asgDimension: "AutoScalingGroupName"
 };
 /** Shared secret generation options (all ClickHouse users share the same policy). */
@@ -295,3 +343,66 @@ export const BACKUP_TASK_MEMORY_MIB = 256;
 export const BACKUP_TASK_CPU_UNITS = 256;
 /** Backup object expiration: 14 days (retains 14 daily snapshots). */
 export const BACKUP_RETENTION_DAYS = 14;
+/** Scratch database the backup task restores into to prove the backup it just
+ *  wrote can be read back. Dropped on the way in and on the way out; never
+ *  queried by the application. */
+export const CLICKHOUSE_BACKUP_SCRATCH_DATABASE = "fjall_backup_verify";
+/** Multiple of the critical free-space floor the verify restore must leave
+ *  clear to proceed.
+ *
+ *  A full restore writes a second copy of the database onto the volume it is
+ *  verifying, so the check has to be bounded by the same quantity the disk
+ *  alarm pages on — otherwise the mechanism that protects the data is also
+ *  the mechanism most likely to fill the disk. Twice the floor rather than
+ *  once so a verify can never itself be the thing that pages: at exactly the
+ *  floor the restore would land the volume on the alarm threshold and hold it
+ *  there until the scratch database is dropped. */
+export const CLICKHOUSE_BACKUP_VERIFY_FREE_SPACE_MARGIN = 2;
+/** Backup task log markers. Distinct full tokens, because the CloudWatch
+ *  metric filters that read them match on substrings — a shared `BACKUP_`
+ *  stem would make the OK filter count the failures too. */
+export const CLICKHOUSE_BACKUP_VERIFY_OK_MARKER = "BACKUP_VERIFY_OK";
+export const CLICKHOUSE_BACKUP_VERIFY_FAILED_MARKER = "BACKUP_VERIFY_FAILED";
+export const CLICKHOUSE_BACKUP_VERIFY_SKIPPED_MARKER = "BACKUP_VERIFY_SKIPPED_FOR_SIZE";
+/** Bounds on the verify restore's wall-clock ceiling, in seconds.
+ *
+ *  Without a ceiling a hung RESTORE is the one backup failure with NO signal
+ *  at all: BACKUP_CREATED has already been emitted so the heartbeat is
+ *  satisfied, and the verify alarms count markers rather than their absence,
+ *  so a task that never finishes reports nothing. `timeout` turns that
+ *  silence into exit 124, which takes the same branch as a raised restore
+ *  and pages.
+ *
+ *  The ceiling scales with the volume (`clickHouseBackupVerifyTimeoutSeconds`
+ *  below) because the largest restore the free-space gate admits scales with
+ *  it: on the default 400 GiB the floor of one hour already covers it, but a
+ *  multi-TiB volume can legitimately need hours, and a fixed hour would page
+ *  FAILED on every healthy verify past the size where the restore outgrows
+ *  it. The 20-hour cap keeps a daily-scheduled verify from overlapping its
+ *  successor — two concurrent runs share the scratch database destructively,
+ *  so the bound must land inside the schedule interval. */
+export const CLICKHOUSE_BACKUP_VERIFY_MIN_TIMEOUT_SECONDS = 3600;
+export const CLICKHOUSE_BACKUP_VERIFY_MAX_TIMEOUT_SECONDS = 72000;
+/** Verify-restore timeout for a given data-volume size.
+ *
+ *  Worst case admitted by the free-space gate is a database of roughly half
+ *  the volume (it must fit beside itself). Writing that back at the gp3
+ *  baseline throughput (`CLICKHOUSE_EBS_THROUGHPUT_MBPS`) bounds the healthy
+ *  duration; doubled for read + merge overhead, then clamped to the
+ *  floor/cap pair above. At the 400 GiB default this resolves to the
+ *  one-hour floor, so the default behaviour is unchanged from the flat bound
+ *  it replaced.
+ *
+ *  Deliberately the BASELINE throughput, not the provisioned `throughputMbps`
+ *  prop. The volume's provisioned figure is an upper bound the instance may
+ *  not reach — an m7g.medium's EBS bandwidth sits under the gp3 baseline, and
+ *  no default-tier host reaches a provisioned 1,000 MiB/s — so a bound that
+ *  tightened with provisioning would page FAILED on healthy restores on
+ *  exactly the hosts customers start on. The baseline-derived bound is loose
+ *  on a provisioned volume, which only delays a hung-restore page; the cap
+ *  keeps that delay inside the schedule interval either way. */
+export function clickHouseBackupVerifyTimeoutSeconds(storageGb) {
+    const worstCaseRestoreMib = (storageGb / 2) * 1024;
+    const derived = Math.ceil((worstCaseRestoreMib / CLICKHOUSE_EBS_THROUGHPUT_MBPS) * 2);
+    return Math.min(CLICKHOUSE_BACKUP_VERIFY_MAX_TIMEOUT_SECONDS, Math.max(CLICKHOUSE_BACKUP_VERIFY_MIN_TIMEOUT_SECONDS, derived));
+}