@fjall/components-infrastructure 16.0.1 → 17.0.0

package/dist/lib/patterns/aws/clickhouseDatabase.d.ts

@@ -29,15 +29,82 @@ export interface ClickHouseDatabaseProps {
      *  taking a new type live needs a launch-template refresh + instance
      *  refresh + ClickHouse restart, not a config reload.
      *
-     *  WARNING — changing this on a DEPLOYED stack REPLACES the persistent
-     *  data volume, not just the instance. The instance type is embedded in
-     *  the compute construct's logical-ID key (`getEc2ConfigKey`), so the
-     *  `AWS::EC2::Volume` beneath it gets a new logical ID: CloudFormation
-     *  snapshots-then-deletes the old volume (`RemovalPolicy.SNAPSHOT`) and
-     *  creates an EMPTY one, and the volume's `ownerLogicalId` re-attach tag
-     *  chain resets. Plan a snapshot-restore migration for live data — this
-     *  is NOT a config-only change on a stack with data. */
+     *  Changing this on a DEPLOYED stack keeps the data. The volume's logical
+     *  ID and its `fjall:OwnerLogicalId` re-attach tag both derive from the
+     *  capacity ANCHOR (`resolveCapacityAnchor`, `${Aws.STACK_NAME}/${anchor}/
+     *  PersistentDataVolume`), not from the instance type, so neither moves
+     *  when the type does and the re-attach chain holds. Config-keyed identity
+     *  was the pre-6.0 derivation — `getEc2ConfigKey` survives only to emit
+     *  the manifest's legacy alias.
+     *
+     *  It is still not free: the type lives in the launch template and in
+     *  user-data-baked server tuning, so it takes an instance refresh, and on
+     *  this single-node cluster that refresh is a restart. The ASG terminates
+     *  the instance (TERMINATING hook detaches the volume), launches a
+     *  replacement, and the LAUNCHING hook re-attaches — ClickHouse is
+     *  unavailable across that window. Size the change into a quiet period;
+     *  do not plan a snapshot-restore migration for it. */
     instanceType?: string;
+    /**
+     * EBS data-volume size in GiB. Default `CLICKHOUSE_EBS_VOLUME_SIZE_GB`
+     * (400). Integer, between `CLICKHOUSE_MIN_STORAGE_GB` and the gp3 ceiling;
+     * validated against `alarms.diskFreeCriticalGib` as a pair, because a
+     * volume smaller than five times that floor would deploy a critical alarm
+     * that can never leave ALARM.
+     *
+     * GROWTH ONLY, and the growth is not instantaneous. EBS `ModifyVolume`
+     * refuses to shrink a volume, so a decrease fails mid-update and rolls the
+     * stack back; it also permits one modification per volume per 6 hours, so
+     * a second bump inside that window fails until the cooldown elapses.
+     *
+     * Resizing the block device is half the job — the ext4 filesystem on it
+     * does not follow until `resize2fs` runs, which is a step in
+     * `buildClickHouseUserData` and therefore runs at BOOT. The declared size
+     * is embedded in that user data, so changing this prop versions the launch
+     * template and triggers the ASG instance refresh whose boot grows the
+     * filesystem — the increase applies in the same deploy, by the same
+     * mechanism an `instanceType` change rides. On this single-node cluster
+     * that relaunch is a restart (the TERMINATING hook detaches the volume,
+     * LAUNCHING re-attaches it): the data survives; the cluster is unavailable
+     * across the window. Size increases into a quiet period, exactly as with
+     * `instanceType`.
+     */
+    storageGb?: number;
+    /**
+     * Provisioned IOPS on the gp3 data volume. Default `CLICKHOUSE_EBS_IOPS`
+     * (3000, the baseline the volume price includes). Integer, between
+     * `CLICKHOUSE_MIN_IOPS` and `CLICKHOUSE_MAX_IOPS`, and at most
+     * `CLICKHOUSE_MAX_IOPS_PER_GB` × `storageGb` — EBS rejects more IOPS than
+     * the volume is large enough to carry. Validated with `throughputMbps` as
+     * a pair, because throughput needs IOPS to ride on (below).
+     *
+     * The same knob RDS exposes as provisioned IOPS on gp3 storage. Applied by
+     * EBS `ModifyVolume` in place: no launch-template change, no instance
+     * relaunch, no restart — the cluster stays up. It shares the
+     * one-modification-per-volume-per-6-hours cooldown with `storageGb` and
+     * `throughputMbps`; a change to any of the three inside that window fails
+     * until it elapses, so move them together.
+     *
+     * Worth raising only once the host can use it: a volume's provisioned
+     * figure is a ceiling the instance's own EBS bandwidth may sit under, so
+     * pair a large value with an `instanceType` whose EBS bandwidth reaches it.
+     */
+    iops?: number;
+    /**
+     * Provisioned throughput on the gp3 data volume, in MiB/s. Default
+     * `CLICKHOUSE_EBS_THROUGHPUT_MBPS` (125, the baseline the volume price
+     * includes). Integer, between `CLICKHOUSE_MIN_THROUGHPUT_MBPS` and
+     * `CLICKHOUSE_MAX_THROUGHPUT_MBPS`, and needing
+     * `CLICKHOUSE_IOPS_PER_THROUGHPUT_MBPS` (4) IOPS per MiB/s — 1000 MiB/s
+     * needs `iops: 4000` or more, and the synth error names the figure.
+     *
+     * The same knob RDS exposes as storage throughput on gp3. Applied in place
+     * by `ModifyVolume` like `iops`, under the same shared 6-hour cooldown, and
+     * likewise bounded by the instance's EBS bandwidth. The restore-verify
+     * timeout is derived from the baseline rather than this value on purpose
+     * (see `clickHouseBackupVerifyTimeoutSeconds`).
+     */
+    throughputMbps?: number;
     /**
      * ECS service desired task count. Defaults to 1 (single-node, single-task
      * by design). Override to 0 via the `clickhouseDesiredCount` CDK context
@@ -57,6 +124,12 @@ export interface ClickHouseDatabaseProps {
      * - omitted/undefined: cold tier ENABLED (S3 bucket provisioned).
      * - `false`: cold tier DISABLED (single-tier hot storage on EBS only).
      * - `{ mode: "s3" }`: cold tier ENABLED (explicit).
+     *
+     * Interacts with the backup restore-verify: the verify restores onto the
+     * EBS volume, so a cold-heavy database can outgrow the free-space gate
+     * long before the hot tier fills — from that day the verify skips (and its
+     * alarm pages daily) with no cure but a larger `storageGb`. When that
+     * trade is deliberate, `backupVerify: false` is the sanctioned off-switch.
      */
     coldTier?: false | {
         mode: "s3";
@@ -85,10 +158,32 @@ export interface ClickHouseDatabaseProps {
      * schedules sparser than ~5.8 days must disable the heartbeat.
      */
     backupSchedule?: string | false;
+    /**
+     * Restore-verify on each scheduled backup. Omitted (the default): after
+     * `BACKUP DATABASE … TO S3` returns, the task restores what it just wrote
+     * into a scratch database, compares the table set, and emits a verdict
+     * marker the verify alarms read — the only signal that the backups are
+     * actually restorable. `false` disables the verify (the backup itself
+     * still runs and the backup-failure alarm + success heartbeat remain);
+     * the verify alarms are withheld with it, so disabling never reads as a
+     * silent green. Reach for it when the restore cost is structurally
+     * unaffordable on-box — a cold-heavy dataset (see `coldTier`), or a
+     * volume deliberately sized tight — and accept that `BACKUP_CREATED`
+     * becomes the only evidence a backup exists.
+     */
+    backupVerify?: false;
     /**
      * Backup-bucket lifecycle expiration in days for both current and noncurrent
      * versions. Default: `BACKUP_RETENTION_DAYS` from `clickhouseConstants.ts`.
      * Ignored when `backupBucket` is supplied (caller-owned lifecycle).
+     * Must exceed the `backupSchedule` interval — a retention shorter than the
+     * gap between runs expires each backup before its successor exists, which
+     * is a window with no backup at all rather than a short history.
+     *
+     * An integer between `CLICKHOUSE_MIN_BACKUP_RETENTION_DAYS` and
+     * `CLICKHOUSE_MAX_BACKUP_RETENTION_DAYS`. This is an S3 object lifetime and
+     * deliberately does NOT share the RDS 35-day ceiling, which bounds a
+     * different mechanism (`backupRetention` on the relational types).
      */
     backupRetentionDays?: number;
     /**

package/dist/lib/patterns/aws/clickhouseDatabase.js

@@ -23,7 +23,10 @@ import { createClickHouseAlarms, validateClickHouseAlarmThresholds, BACKUP_HEART
 import { ClickHouseSchemaAdminSchema, ManagedPasswordNameSchema, ProfileSpecSchema, PROFILE_NAME_PATTERN } from "../../resources/aws/database/clickhouseSchemas.js";
 import { deriveClickHouseDefaultProfiles } from "../../resources/aws/database/clickhouseTuning.js";
 import { inferAmiHardwareType } from "../../resources/aws/compute/ecsConstants.js";
-import { CLICKHOUSE_DATABASE_NAME, DEFAULT_CLICKHOUSE_INSTANCE_TYPE, CLICKHOUSE_IMAGE, CLICKHOUSE_EBS_VOLUME_SIZE_GB, CLICKHOUSE_EBS_IOPS, CLICKHOUSE_EBS_THROUGHPUT_MBPS, clickHouseTaskMemoryMiB, CLICKHOUSE_HTTP_PORT, CLICKHOUSE_HTTPS_PORT, CLICKHOUSE_NATIVE_PORT, CLICKHOUSE_TCP_SECURE_PORT, CLICKHOUSE_TLS_CERT_MOUNT_PATH, CLICKHOUSE_PROMETHEUS_PORT, CLICKHOUSE_DATA_MOUNT_PATH, CLICKHOUSE_SECRET_OPTIONS, CLICKHOUSE_SERVER_ROLE_TAG, CLICKHOUSE_HOST_METRICS, clickHouseUserSecretName, CLICKHOUSE_HEALTH_CHECK, CLICKHOUSE_STOP_TIMEOUT_SECONDS, CLICKHOUSE_EBS_DEVICE_NAME, CLICKHOUSE_CONFIG_SUBDIR, CLICKHOUSE_USERS_SUBDIR, userPasswordEnvName, OPTIMISE_FINAL_SCHEDULE, REPLACING_MERGE_TREE_TABLES, OPTIMISE_MV_TABLES, CLICKHOUSE_CLOUDMAP_SERVICE_NAME, CLICKHOUSE_SERVER_CONTAINER_NAME, CLICKHOUSE_SERVICE_NAME, OPTIMISE_TASK_MEMORY_MIB, OPTIMISE_TASK_CPU_UNITS, BACKUP_SCHEDULE, BACKUP_TASK_MEMORY_MIB, BACKUP_TASK_CPU_UNITS, BACKUP_RETENTION_DAYS } from "../../resources/aws/database/clickhouseConstants.js";
+import { CLICKHOUSE_DATABASE_NAME, DEFAULT_CLICKHOUSE_INSTANCE_TYPE, CLICKHOUSE_IMAGE, clickHouseTaskMemoryMiB, CLICKHOUSE_HTTP_PORT, CLICKHOUSE_HTTPS_PORT, CLICKHOUSE_NATIVE_PORT, CLICKHOUSE_TCP_SECURE_PORT, CLICKHOUSE_TLS_CERT_MOUNT_PATH, CLICKHOUSE_PROMETHEUS_PORT, CLICKHOUSE_DATA_MOUNT_PATH, CLICKHOUSE_SECRET_OPTIONS, CLICKHOUSE_SERVER_ROLE_TAG, CLICKHOUSE_HOST_METRICS, clickHouseUserSecretName, CLICKHOUSE_HEALTH_CHECK, CLICKHOUSE_STOP_TIMEOUT_SECONDS, CLICKHOUSE_EBS_DEVICE_NAME, CLICKHOUSE_CONFIG_SUBDIR, CLICKHOUSE_USERS_SUBDIR, userPasswordEnvName, OPTIMISE_FINAL_SCHEDULE, REPLACING_MERGE_TREE_TABLES, OPTIMISE_MV_TABLES, CLICKHOUSE_CLOUDMAP_SERVICE_NAME, CLICKHOUSE_SERVER_CONTAINER_NAME, CLICKHOUSE_SERVICE_NAME, OPTIMISE_TASK_MEMORY_MIB, OPTIMISE_TASK_CPU_UNITS, BACKUP_SCHEDULE, BACKUP_TASK_MEMORY_MIB, BACKUP_TASK_CPU_UNITS, BACKUP_RETENTION_DAYS } from "../../resources/aws/database/clickhouseConstants.js";
+import { CLICKHOUSE_MAX_BACKUP_RETENTION_DAYS, CLICKHOUSE_MIN_BACKUP_RETENTION_DAYS } from "@fjall/util/clickhouse";
+import { resolveClickHouseStorage, resolveClickHouseVolumePerformance } from "../../resources/aws/database/clickhouseStorage.js";
+import { buildClickHouseBackupScript } from "../../resources/aws/database/clickhouseBackupScript.js";
 import { TlsCertGenerator } from "../../resources/aws/utilities/tlsCertGenerator.js";
 import { EcsCompute } from "./computeEcs.js";
 /**
@@ -263,6 +266,7 @@ export class ClickHouseDatabase extends Construct {
         const desiredCount = resolveClickHouseDesiredCount(this.node.tryGetContext("clickhouseDesiredCount"), props.desiredCount);
         const optimiseEnabled = props.optimiseSchedule !== false;
         const backupEnabled = props.backupSchedule !== false;
+        const backupVerifyEnabled = props.backupVerify !== false;
         const optimiseSchedule = typeof props.optimiseSchedule === "string"
             ? props.optimiseSchedule
             : OPTIMISE_FINAL_SCHEDULE;
@@ -270,9 +274,15 @@ export class ClickHouseDatabase extends Construct {
             ? props.backupSchedule
             : BACKUP_SCHEDULE;
         const backupRetentionDays = props.backupRetentionDays ?? BACKUP_RETENTION_DAYS;
-        if (props.backupRetentionDays !== undefined &&
-            (props.backupRetentionDays < 1 || props.backupRetentionDays > 3650)) {
-            throw new Error(`ClickHouseDatabase: backupRetentionDays must be between 1 and 3650; got ${props.backupRetentionDays}.`);
+        // Checked after defaulting, so the default is held to the same bound as a
+        // caller's value. Integer-ness is part of the bound rather than a separate
+        // nicety: the value becomes `Duration.days()` on an S3 lifecycle rule,
+        // which rejects a fraction far downstream with no mention of the knob that
+        // supplied it.
+        if (!Number.isInteger(backupRetentionDays) ||
+            backupRetentionDays < CLICKHOUSE_MIN_BACKUP_RETENTION_DAYS ||
+            backupRetentionDays > CLICKHOUSE_MAX_BACKUP_RETENTION_DAYS) {
+            throw new Error(`ClickHouseDatabase: backupRetentionDays must be an integer between ${CLICKHOUSE_MIN_BACKUP_RETENTION_DAYS} and ${CLICKHOUSE_MAX_BACKUP_RETENTION_DAYS} days; got ${backupRetentionDays}.`);
         }
         const tlsOptions = props.tls ?? {
             mode: "self-signed"
@@ -348,10 +358,29 @@ export class ClickHouseDatabase extends Construct {
                 retention: RetentionDays.TWO_WEEKS
             })
             : undefined;
+        // Resolved BEFORE the user data is built: the declared volume size is
+        // embedded into the script so a storageGb change versions the launch
+        // template (see `dataVolumeSizeGb` on BuildClickHouseUserDataOptions).
+        // The restore-verify gate is bounded by the same free-space floor the
+        // disk alarm pages on — the check that protects the data must not be
+        // able to fill the volume it is protecting. Volume size and floor are
+        // only checkable against each other, and the floor lives on the alarms
+        // knob.
+        const alarmThresholds = typeof props.alarms === "object" ? props.alarms : {};
+        validateClickHouseAlarmThresholds(alarmThresholds);
+        const { storageGb, diskFreeCriticalGib } = resolveClickHouseStorage(props.storageGb, alarmThresholds.diskFreeCriticalGib);
+        // IOPS and throughput are bounded by each other and by the size just
+        // resolved, so they resolve second, against the validated storageGb.
+        const { iops, throughputMbps } = resolveClickHouseVolumePerformance({
+            storageGb,
+            iops: props.iops,
+            throughputMbps: props.throughputMbps
+        });
         const userData = UserData.custom(buildClickHouseUserData({
             backupBucketName: backupBucket.bucketName,
             backupBucketRegion: Stack.of(this).region,
             instanceType,
+            dataVolumeSizeGb: storageGb,
             ...(coldTierBucket !== undefined && {
                 coldTier: {
                     bucketName: coldTierBucket.bucketName,
@@ -405,12 +434,15 @@ export class ClickHouseDatabase extends Construct {
         // deploy-order window, or a dropped table lingering in the constants)
         // would permanently skip every optimise after it. Failures are logged and
         // the task still exits non-zero so they stay visible in the task history.
-        const optimiseClient = `clickhouse-client --host ${clickHouseHost} --port ${nativePort} --user ${schemaAdmin.name}${sidecarTlsClientArgs}`;
+        // Both scheduled tasks authenticate identically; the password arrives via
+        // CLICKHOUSE_PASSWORD rather than argv.
+        const clickHouseClient = `clickhouse-client --host ${clickHouseHost} --port ${nativePort} --user ${schemaAdmin.name}${sidecarTlsClientArgs}`;
         const optimiseScript = [
             "FAILED=0",
-            ...optimiseStatements.map((stmt) => `${optimiseClient} --query "${stmt}" || { echo "optimise failed: ${stmt}"; FAILED=1; }`),
+            ...optimiseStatements.map((stmt) => `${clickHouseClient} --query "${stmt}" || { echo "optimise failed: ${stmt}"; FAILED=1; }`),
             'exit "$FAILED"'
         ].join("; ");
+        // Resolved before the scheduled tasks because the backup task's
         const scheduledTasks = [];
         if (optimiseEnabled) {
             scheduledTasks.push({
@@ -439,7 +471,14 @@ export class ClickHouseDatabase extends Construct {
                     "sh",
                     "-c",
                     // Password via CLICKHOUSE_PASSWORD env, not --password on argv (argv → /proc/<pid>/cmdline).
-                    `${sidecarTlsPreamble}STAMP=$(date +%Y%m%d-%H%M%S) && clickhouse-client --host ${clickHouseHost} --port ${nativePort} --user ${schemaAdmin.name}${sidecarTlsClientArgs} --query "BACKUP DATABASE ${CLICKHOUSE_DATABASE_NAME} TO S3('${backupDestUrl}weekly-$STAMP/')"`
+                    `${sidecarTlsPreamble}${buildClickHouseBackupScript({
+                        client: clickHouseClient,
+                        databaseName: CLICKHOUSE_DATABASE_NAME,
+                        backupDestUrl,
+                        diskFreeCriticalGib,
+                        storageGb,
+                        verify: backupVerifyEnabled
+                    })}`
                 ],
                 secrets: {
                     CLICKHOUSE_PASSWORD: EcsSecret.fromSecretsManager(adminSecret.secret, "password"),
@@ -459,8 +498,6 @@ export class ClickHouseDatabase extends Construct {
         const alarmsDisabled = props.alarms === false;
         const applicationId = props.applicationId ??
             (App.hasInstance() ? App.getInstance().getName() : undefined);
-        const alarmThresholds = typeof props.alarms === "object" ? props.alarms : {};
-        validateClickHouseAlarmThresholds(alarmThresholds);
         const serviceLogAlarms = alarmsDisabled
             ? undefined
             : buildClickHouseLogAlarmSpecs(alarmThresholds, coldTierEnabled);
@@ -520,11 +557,11 @@ export class ClickHouseDatabase extends Construct {
                         instanceMonitoring: Monitoring.BASIC,
                         availabilityZones: [dataAz],
                         persistentDataVolume: {
-                            sizeGb: CLICKHOUSE_EBS_VOLUME_SIZE_GB,
+                            sizeGb: storageGb,
                             deviceName: CLICKHOUSE_EBS_DEVICE_NAME,
                             availabilityZone: dataAz,
-                            iops: CLICKHOUSE_EBS_IOPS,
-                            throughputMbps: CLICKHOUSE_EBS_THROUGHPUT_MBPS,
+                            iops,
+                            throughputMbps,
                             ...(resolvedAlertsTopic !== undefined && {
                                 alarmTopic: resolvedAlertsTopic
                             })
@@ -658,6 +695,8 @@ export class ClickHouseDatabase extends Construct {
                 asgName,
                 alarmTopic: resolvedAlertsTopic,
                 config: alarmThresholds,
+                diskFreeCriticalGib,
+                backupVerifyEnabled,
                 ...(applicationId !== undefined && { applicationId }),
                 ...(backupTaskLogGroup !== undefined && { backupTaskLogGroup })
             });

package/dist/lib/patterns/aws/database.d.ts

@@ -332,7 +332,7 @@ export declare class RelationalDatabase extends Construct implements IRelational
     grantIamConnect(grantee: IGrantable, dbUsername: string): Grant;
 }
 export { ClickHouseDatabase, type ClickHouseDatabaseProps, type ClickHouseAlarmThresholds };
-export { ClickHouseDefaultProfiles, ClickHouseSchemaAdminSchema, ManagedPasswordNameSchema, ProfileSpecSchema, type ClickHouseSchemaAdmin, type ManagedPasswordName, type ProfileSpec } from "../../resources/aws/database/clickhouseSchemas.js";
+export { ClickHouseDefaultProfiles, ClickHouseSchemaAdminSchema, ManagedPasswordNameSchema, ProfileSpecSchema, SettingConstraintSchema, type ClickHouseSchemaAdmin, type FrozenProfileSpec, type ManagedPasswordName, type ProfileSpec, type SettingConstraint } from "../../resources/aws/database/clickhouseSchemas.js";
 export { renderUsersXml } from "../../resources/aws/database/clickhouseXmlRenderer.js";
 export type { RenderUsersXmlOptions } from "../../resources/aws/database/clickhouseXmlRenderer.js";
 export type { DynamoDBKeySchema, DynamoDBGlobalSecondaryIndex, DynamoDBTableProps } from "../../resources/aws/database/dynamodb.js";

package/dist/lib/patterns/aws/database.js

@@ -655,6 +655,6 @@ export class RelationalDatabase extends Construct {
     }
 }
 export { ClickHouseDatabase };
-export { ClickHouseDefaultProfiles, ClickHouseSchemaAdminSchema, ManagedPasswordNameSchema, ProfileSpecSchema } from "../../resources/aws/database/clickhouseSchemas.js";
+export { ClickHouseDefaultProfiles, ClickHouseSchemaAdminSchema, ManagedPasswordNameSchema, ProfileSpecSchema, SettingConstraintSchema } from "../../resources/aws/database/clickhouseSchemas.js";
 export { renderUsersXml } from "../../resources/aws/database/clickhouseXmlRenderer.js";
 export { isRelationalDatabase, isDynamoDBDatabase, isClickHouseDatabase, isAuroraDatabase, isInstanceDatabase, isGlobalAuroraDatabase } from "./interfaces/database.js";

package/dist/lib/resources/aws/database/clickhouseBackupScript.d.ts

@@ -0,0 +1,103 @@
+/**
+ * The backup task's shell body: take the backup, then prove it restores.
+ *
+ * `BACKUP DATABASE … TO S3(…)` returning `BACKUP_CREATED` means ClickHouse
+ * finished writing and the statement did not raise. It does NOT mean the
+ * bytes it wrote can be read back — and the gap between those two is the
+ * whole reason a backup exists. So the task restores what it just wrote,
+ * into a scratch database, and says so in its own log line.
+ *
+ * The restore is a FULL one, deliberately. `SETTINGS structure_only = 1`
+ * looks like the cheap version of this check and is not a version of it at
+ * all: on 26.3.17.56 a structure-only restore of a backup whose entire
+ * `data/` tree had been deleted returned `RESTORED` and exit 0, because it
+ * reads `metadata/` and never opens a part. A verification that passes on a
+ * backup with no data in it is worse than no verification — it converts an
+ * unrestorable backup into a green signal. The full restore fails that same
+ * case loudly (`STD_EXCEPTION`, missing `checksums.txt`), which is the
+ * behaviour the alarm is built on.
+ */
+export interface ClickHouseBackupScriptParams {
+    /** Fully-formed `clickhouse-client …` invocation, minus `--query`. */
+    readonly client: string;
+    /** Database the backup covers, and the restore compares against. */
+    readonly databaseName: string;
+    /** `https://<bucket>.s3.<region>.amazonaws.com/backup/` — trailing slash. */
+    readonly backupDestUrl: string;
+    /** Critical free-space floor in GiB; the verify gate keeps twice this
+     *  clear so the check can never be the thing that pages. */
+    readonly diskFreeCriticalGib: number;
+    /** Declared data-volume size in GiB; bounds the verify restore's timeout
+     *  via `clickHouseBackupVerifyTimeoutSeconds`. */
+    readonly storageGb: number;
+    /** When false, the script takes the backup and stops — no restore verify.
+     *  The caller (`ClickHouseDatabase`) also withholds the verify alarms, so
+     *  the two halves of the contract disable together. */
+    readonly verify: boolean;
+}
+/**
+ * Builds the backup + restore-verify script.
+ *
+ * Statement order carries decisions that are not obvious from reading it
+ * forward:
+ *
+ * 1. **The scratch database is dropped FIRST, not only last.** A restore
+ *    onto an existing database of that name succeeds silently — verified on
+ *    26.3.17.56 — so a run that died between RESTORE and DROP would leave
+ *    tables behind that satisfy the next run's table-set comparison no
+ *    matter what the new backup contains. Dropping on the way in also makes
+ *    the free-space reading below honest, since a leftover scratch copy is
+ *    occupying the disk being measured.
+ *
+ * 2. **The live row count is sampled BEFORE the backup, and it arms the
+ *    zero-rows check.** A restore of a genuinely empty database restores
+ *    zero rows and is CORRECT; failing it would page FAILED on every fresh
+ *    stack from night one. So `restored_rows=0` is a failure only when the
+ *    source had rows at backup time. A failed source sample reads as
+ *    `unknown`, which arms the check — an unreadable source is not evidence
+ *    the database was empty (fail closed, per the read-failure posture).
+ *
+ * 3. **The restore is gated on free space, and the gate reads the same
+ *    quantity the critical disk alarm guards.** A full restore writes a
+ *    second copy of the database onto the volume it is verifying. That is
+ *    affordable at today's size and stops being affordable at some larger
+ *    one, and the failure mode if it is left ungated is the worst kind: the
+ *    backup check fills the disk that the backups exist to protect. When
+ *    the copy would not leave `diskFreeCriticalGib × margin` clear, the run
+ *    skips the restore and SAYS SO with its own marker. Silent degradation
+ *    is not available here — a skipped verify that logged nothing would be
+ *    indistinguishable from a passing one. The same discipline covers the
+ *    probe itself: a failed free-space read emits the SKIPPED marker (the
+ *    verify did not run, which is exactly what that marker means) and exits
+ *    non-zero, rather than being the one silent path out of the script.
+ *
+ * 4. **The restore is bounded by `timeout`, sized from the volume.** A hung
+ *    restore is the one backup failure with no signal: `BACKUP_CREATED` is
+ *    already emitted so the heartbeat is satisfied, and the verify alarms
+ *    count markers rather than their absence. The bound converts that
+ *    silence into exit 124, which takes the same branch as a raised
+ *    restore. It scales with `storageGb` because the largest restore the
+ *    free-space gate admits scales with the volume — a flat bound would
+ *    misreport every healthy verify as FAILED past the size where the
+ *    restore outgrows it.
+ *
+ * 5. **The comparison is on the table SET, never on row counts — and it
+ *    ignores materialised-view inner tables.** The database keeps ingesting
+ *    during and after the backup, so the restored copy is a point-in-time
+ *    from somewhere inside the backup window and can never be expected to
+ *    match live counts. Rows are checked only against zero (armed per 2).
+ *    MV inner tables (`.inner_id.<uuid>` on Atomic databases) are excluded
+ *    from both sides of the set comparison: `RESTORE … AS <scratch>` on the
+ *    same server assigns NEW UUIDs, so the live database's inner names can
+ *    never appear in the scratch database and a raw name comparison reports
+ *    every implicit MV as missing — a FAILED page against a good backup.
+ *    The MVs themselves (and their data, via the restored inner tables)
+ *    remain fully compared.
+ *
+ * Every statement carries an explicit `|| …` failure branch rather than
+ * relying on `set -e`. The TLS preamble this script is concatenated onto
+ * sets `set -eu`, but ONLY when TLS is active — so a script depending on
+ * `set -e` would abort on the first failure with TLS on and blunder past it
+ * with TLS off. Explicit branches behave identically either way.
+ */
+export declare function buildClickHouseBackupScript(params: ClickHouseBackupScriptParams): string;

package/dist/lib/resources/aws/database/clickhouseBackupScript.js

@@ -0,0 +1,123 @@
+/**
+ * The backup task's shell body: take the backup, then prove it restores.
+ *
+ * `BACKUP DATABASE … TO S3(…)` returning `BACKUP_CREATED` means ClickHouse
+ * finished writing and the statement did not raise. It does NOT mean the
+ * bytes it wrote can be read back — and the gap between those two is the
+ * whole reason a backup exists. So the task restores what it just wrote,
+ * into a scratch database, and says so in its own log line.
+ *
+ * The restore is a FULL one, deliberately. `SETTINGS structure_only = 1`
+ * looks like the cheap version of this check and is not a version of it at
+ * all: on 26.3.17.56 a structure-only restore of a backup whose entire
+ * `data/` tree had been deleted returned `RESTORED` and exit 0, because it
+ * reads `metadata/` and never opens a part. A verification that passes on a
+ * backup with no data in it is worse than no verification — it converts an
+ * unrestorable backup into a green signal. The full restore fails that same
+ * case loudly (`STD_EXCEPTION`, missing `checksums.txt`), which is the
+ * behaviour the alarm is built on.
+ */
+import { CLICKHOUSE_BACKUP_SCRATCH_DATABASE, CLICKHOUSE_BACKUP_VERIFY_FREE_SPACE_MARGIN, CLICKHOUSE_BACKUP_VERIFY_FAILED_MARKER, CLICKHOUSE_BACKUP_VERIFY_OK_MARKER, CLICKHOUSE_BACKUP_VERIFY_SKIPPED_MARKER, clickHouseBackupVerifyTimeoutSeconds } from "./clickhouseConstants.js";
+const BYTES_PER_GIB = 1024 ** 3;
+/**
+ * Builds the backup + restore-verify script.
+ *
+ * Statement order carries decisions that are not obvious from reading it
+ * forward:
+ *
+ * 1. **The scratch database is dropped FIRST, not only last.** A restore
+ *    onto an existing database of that name succeeds silently — verified on
+ *    26.3.17.56 — so a run that died between RESTORE and DROP would leave
+ *    tables behind that satisfy the next run's table-set comparison no
+ *    matter what the new backup contains. Dropping on the way in also makes
+ *    the free-space reading below honest, since a leftover scratch copy is
+ *    occupying the disk being measured.
+ *
+ * 2. **The live row count is sampled BEFORE the backup, and it arms the
+ *    zero-rows check.** A restore of a genuinely empty database restores
+ *    zero rows and is CORRECT; failing it would page FAILED on every fresh
+ *    stack from night one. So `restored_rows=0` is a failure only when the
+ *    source had rows at backup time. A failed source sample reads as
+ *    `unknown`, which arms the check — an unreadable source is not evidence
+ *    the database was empty (fail closed, per the read-failure posture).
+ *
+ * 3. **The restore is gated on free space, and the gate reads the same
+ *    quantity the critical disk alarm guards.** A full restore writes a
+ *    second copy of the database onto the volume it is verifying. That is
+ *    affordable at today's size and stops being affordable at some larger
+ *    one, and the failure mode if it is left ungated is the worst kind: the
+ *    backup check fills the disk that the backups exist to protect. When
+ *    the copy would not leave `diskFreeCriticalGib × margin` clear, the run
+ *    skips the restore and SAYS SO with its own marker. Silent degradation
+ *    is not available here — a skipped verify that logged nothing would be
+ *    indistinguishable from a passing one. The same discipline covers the
+ *    probe itself: a failed free-space read emits the SKIPPED marker (the
+ *    verify did not run, which is exactly what that marker means) and exits
+ *    non-zero, rather than being the one silent path out of the script.
+ *
+ * 4. **The restore is bounded by `timeout`, sized from the volume.** A hung
+ *    restore is the one backup failure with no signal: `BACKUP_CREATED` is
+ *    already emitted so the heartbeat is satisfied, and the verify alarms
+ *    count markers rather than their absence. The bound converts that
+ *    silence into exit 124, which takes the same branch as a raised
+ *    restore. It scales with `storageGb` because the largest restore the
+ *    free-space gate admits scales with the volume — a flat bound would
+ *    misreport every healthy verify as FAILED past the size where the
+ *    restore outgrows it.
+ *
+ * 5. **The comparison is on the table SET, never on row counts — and it
+ *    ignores materialised-view inner tables.** The database keeps ingesting
+ *    during and after the backup, so the restored copy is a point-in-time
+ *    from somewhere inside the backup window and can never be expected to
+ *    match live counts. Rows are checked only against zero (armed per 2).
+ *    MV inner tables (`.inner_id.<uuid>` on Atomic databases) are excluded
+ *    from both sides of the set comparison: `RESTORE … AS <scratch>` on the
+ *    same server assigns NEW UUIDs, so the live database's inner names can
+ *    never appear in the scratch database and a raw name comparison reports
+ *    every implicit MV as missing — a FAILED page against a good backup.
+ *    The MVs themselves (and their data, via the restored inner tables)
+ *    remain fully compared.
+ *
+ * Every statement carries an explicit `|| …` failure branch rather than
+ * relying on `set -e`. The TLS preamble this script is concatenated onto
+ * sets `set -eu`, but ONLY when TLS is active — so a script depending on
+ * `set -e` would abort on the first failure with TLS on and blunder past it
+ * with TLS off. Explicit branches behave identically either way.
+ */
+export function buildClickHouseBackupScript(params) {
+    const { client, databaseName, backupDestUrl, diskFreeCriticalGib, storageGb, verify } = params;
+    const backupStatements = [
+        "STAMP=$(date +%Y%m%d-%H%M%S)",
+        `DEST="${backupDestUrl}backup-$STAMP/"`,
+        `${client} --query "BACKUP DATABASE ${databaseName} TO S3('$DEST')" || exit 1`
+    ];
+    if (!verify) {
+        return backupStatements.join("; ");
+    }
+    const scratch = CLICKHOUSE_BACKUP_SCRATCH_DATABASE;
+    const requiredFreeBytes = Math.round(diskFreeCriticalGib *
+        BYTES_PER_GIB *
+        CLICKHOUSE_BACKUP_VERIFY_FREE_SPACE_MARGIN);
+    const verifyTimeoutSeconds = clickHouseBackupVerifyTimeoutSeconds(storageGb);
+    const dropScratch = `${client} --query "DROP DATABASE IF EXISTS ${scratch} SYNC"`;
+    const sourceRowsQuery = `SELECT sum(rows) FROM system.parts WHERE database = '${databaseName}' AND active`;
+    const fitVerdictQuery = `SELECT if((SELECT free_space FROM system.disks WHERE name = 'default')` +
+        ` - (SELECT sum(bytes_on_disk) FROM system.parts WHERE database = '${databaseName}' AND active)` +
+        ` >= ${requiredFreeBytes}, 'FITS', 'SKIP')`;
+    const missingTablesQuery = `SELECT count() FROM (SELECT name FROM system.tables WHERE database = '${databaseName}' AND name NOT LIKE '.inner%'` +
+        ` EXCEPT SELECT name FROM system.tables WHERE database = '${scratch}' AND name NOT LIKE '.inner%')`;
+    const restoredRowsQuery = `SELECT sum(rows) FROM system.parts WHERE database = '${scratch}' AND active`;
+    return [
+        `${dropScratch} || exit 1`,
+        `SRCROWS=$(${client} --query "${sourceRowsQuery}") || SRCROWS=unknown`,
+        ...backupStatements,
+        `FIT=$(${client} --query "${fitVerdictQuery}") || { echo "${CLICKHOUSE_BACKUP_VERIFY_SKIPPED_MARKER} $DEST free-space probe failed — verify did not run"; exit 1; }`,
+        `if [ "$FIT" != "FITS" ]; then echo "${CLICKHOUSE_BACKUP_VERIFY_SKIPPED_MARKER} $DEST restore would not leave ${requiredFreeBytes} bytes free"; exit 0; fi`,
+        `timeout ${verifyTimeoutSeconds} ${client} --query "RESTORE DATABASE ${databaseName} AS ${scratch} FROM S3('$DEST')" || { echo "${CLICKHOUSE_BACKUP_VERIFY_FAILED_MARKER} $DEST restore raised or exceeded ${verifyTimeoutSeconds}s"; ${dropScratch} || true; exit 1; }`,
+        `MISSING=$(${client} --query "${missingTablesQuery}") || MISSING=unknown`,
+        `ROWS=$(${client} --query "${restoredRowsQuery}") || ROWS=0`,
+        `${dropScratch} || true`,
+        `if [ "$MISSING" != "0" ] || { [ "$ROWS" = "0" ] && [ "$SRCROWS" != "0" ]; }; then echo "${CLICKHOUSE_BACKUP_VERIFY_FAILED_MARKER} $DEST missing_tables=$MISSING restored_rows=$ROWS source_rows=$SRCROWS"; exit 1; fi`,
+        `echo "${CLICKHOUSE_BACKUP_VERIFY_OK_MARKER} $DEST restored_rows=$ROWS source_rows=$SRCROWS"`
+    ].join("; ");
+}