@fjall/components-infrastructure 16.0.0 → 17.0.0

package/dist/lib/resources/aws/database/clickhouseTuning.d.ts

@@ -1,4 +1,4 @@
-import type { ProfileSpec } from "./clickhouseSchemas.js";
+import type { FrozenProfileSpec } from "./clickhouseSchemas.js";
 /**
  * Server-level tuning values derived from the instance type. Consumed by
  * `generateServerConfigXml` — every value here lands in the rendered
@@ -50,10 +50,9 @@ export declare const SYSTEM_LOG_TTL = "event_date + INTERVAL 14 DAY DELETE";
 export declare function deriveClickHouseServerTuning(instanceType: string): ClickHouseServerTuning;
 /**
  * Derive the four workload-class default profiles from the instance type.
- * Memory caps, external-spill thresholds and row-read ceilings scale with
- * instance memory; thread caps scale with vCPUs. At the default
- * m7g.medium this reproduces the historical hand-tuned values exactly
- * (pinned by clickhouseTuning.test.ts).
+ * Memory caps, external-spill thresholds and constraint ceilings scale with
+ * instance memory; thread caps scale with vCPUs. The full m7g.medium
+ * shape is pinned by clickhouseTuning.test.ts.
  *
  * `high_throughput_ingest.maxConcurrentQueriesForUser` deliberately does
  * NOT scale: it mirrors the webapp's client-side FIFO gate
@@ -61,4 +60,4 @@ export declare function deriveClickHouseServerTuning(instanceType: string): Clic
  * parity test on the webapp side enforces the pair, so the value is a
  * client contract, not an instance-capacity knob.
  */
-export declare function deriveClickHouseDefaultProfiles(instanceType: string): Readonly<Record<string, Readonly<ProfileSpec>>>;
+export declare function deriveClickHouseDefaultProfiles(instanceType: string): Readonly<Record<string, FrozenProfileSpec>>;

package/dist/lib/resources/aws/database/clickhouseTuning.js

@@ -1,4 +1,4 @@
-import { CLICKHOUSE_INSTANCE_SPECS } from "./clickhouseConstants.js";
+import { clickHouseInstanceSpec, CLICKHOUSE_INSTANCE_SPECS } from "./clickhouseConstants.js";
 /** 4 GiB / 1 vCPU (m7g.medium) baseline the scaling formulas anchor to.
  *  These are the proven prod values the construct shipped hand-tuned before
  *  derivation existed — `deriveClickHouseServerTuning("m7g.medium")` must
@@ -47,7 +47,7 @@ export const MAX_SERVER_MEMORY_RAM_RATIO = 0.75;
  *  change is a single edit. */
 export const SYSTEM_LOG_TTL = "event_date + INTERVAL 14 DAY DELETE";
 function resolveSpec(instanceType) {
-    const spec = CLICKHOUSE_INSTANCE_SPECS[instanceType];
+    const spec = clickHouseInstanceSpec(instanceType);
     if (spec === undefined) {
         throw new Error(`ClickHouseDatabase: unknown instance type '${instanceType}' — cannot ` +
             "derive server tuning. Supported: " +
@@ -110,10 +110,9 @@ export function deriveClickHouseServerTuning(instanceType) {
 }
 /**
  * Derive the four workload-class default profiles from the instance type.
- * Memory caps, external-spill thresholds and row-read ceilings scale with
- * instance memory; thread caps scale with vCPUs. At the default
- * m7g.medium this reproduces the historical hand-tuned values exactly
- * (pinned by clickhouseTuning.test.ts).
+ * Memory caps, external-spill thresholds and constraint ceilings scale with
+ * instance memory; thread caps scale with vCPUs. The full m7g.medium
+ * shape is pinned by clickhouseTuning.test.ts.
  *
  * `high_throughput_ingest.maxConcurrentQueriesForUser` deliberately does
  * NOT scale: it mirrors the webapp's client-side FIFO gate
@@ -124,12 +123,27 @@ export function deriveClickHouseServerTuning(instanceType) {
 export function deriveClickHouseDefaultProfiles(instanceType) {
     const spec = resolveSpec(instanceType);
     const scale = memoryScale(spec);
+    // The four unraisable ceilings — one source for both the ingest profile's
+    // scalar defaults and its <constraints> block, so a retuned default can
+    // never drift from (or exceed) its own ceiling.
+    const ingestCeilings = Object.freeze({
+        maxThreads: spec.vcpus,
+        maxMemoryUsage: 1610612736 * scale, // 1.5 GiB baseline
+        maxMemoryUsageForUser: 2147483648 * scale, // 2 GiB baseline
+        maxExecutionTime: 30
+    });
     return Object.freeze({
         high_throughput_ingest: Object.freeze({
-            maxThreads: spec.vcpus,
+            maxThreads: ingestCeilings.maxThreads,
             maxInsertThreads: spec.vcpus,
             maxConcurrentQueriesForUser: 6,
-            logQueriesMinQueryDurationMs: 100,
+            // MUST stay 0: any non-zero value drops ALL ExceptionBeforeStart rows
+            // (quota / concurrency / limit refusals) and every QueryStart row from
+            // system.query_log unconditionally (executeQuery.cpp gates on elapsed
+            // time, which refused queries never accrue) — failed product queries
+            // become invisible. If log volume ever needs a lever it is
+            // log_queries_min_type=QUERY_FINISH, never a duration floor.
+            logQueriesMinQueryDurationMs: 0,
             optimizeMoveToPrewhere: true,
             useQueryConditionCache: true,
             useSkipIndexesIfFinal: true,
@@ -146,16 +160,25 @@ export function deriveClickHouseDefaultProfiles(instanceType) {
             inputFormatParallelParsing: false,
             outputFormatParallelFormatting: false,
             queryPlanOptimizeLazyMaterialization: true,
-            maxMemoryUsage: 1610612736 * scale, // 1.5 GiB baseline
-            maxMemoryUsageForUser: 2147483648 * scale, // 2 GiB baseline
+            maxMemoryUsage: ingestCeilings.maxMemoryUsage,
+            maxMemoryUsageForUser: ingestCeilings.maxMemoryUsageForUser,
             maxBytesBeforeExternalSort: 536870912 * scale, // 512 MiB baseline
             maxBytesBeforeExternalGroupBy: 536870912 * scale,
-            maxExecutionTime: 30,
-            // The ceiling an unbounded consumer query dies against (TOO_MANY_ROWS).
-            // Scales with memory so a bigger node's tables get proportionate
-            // headroom — but consumers must still time-bound their scans; this is
-            // a guard rail, not a budget to grow into.
-            maxRowsToRead: 10000000 * scale
+            maxExecutionTime: ingestCeilings.maxExecutionTime,
+            // Deliberately no maxRowsToRead: the planning-time estimate refuses
+            // legitimate in-order LIMIT reads, and any statement can override it.
+            // Per-box protection is the unraisable ceilings below; per-tenant
+            // fairness is the org-keyed quota (webapp clickhouse-init/030).
+            constraints: Object.freeze({
+                maxExecutionTime: Object.freeze({
+                    max: ingestCeilings.maxExecutionTime
+                }),
+                maxMemoryUsage: Object.freeze({ max: ingestCeilings.maxMemoryUsage }),
+                maxMemoryUsageForUser: Object.freeze({
+                    max: ingestCeilings.maxMemoryUsageForUser
+                }),
+                maxThreads: Object.freeze({ max: ingestCeilings.maxThreads })
+            })
         }),
         audit_append: Object.freeze({
             maxThreads: 1,

package/dist/lib/resources/aws/database/clickhouseUserData.d.ts

@@ -18,6 +18,17 @@ export interface BuildClickHouseUserDataOptions {
      * as `clickHouseTaskMemoryMiB`).
      */
     instanceType: string;
+    /**
+     * Declared data-volume size in GiB, embedded into the emitted script as a
+     * comment line. Load-bearing, not documentation: user data is part of the
+     * launch template, so embedding the size means a `storageGb` change
+     * produces a new launch-template version — which is what triggers the ASG
+     * instance refresh whose boot runs the `resize2fs` below. Without it, a
+     * volume increase modifies the EBS device in place, nothing relaunches,
+     * and the grown capacity is inert until some unrelated deploy happens to
+     * replace the instance. Same apply mechanism `instanceType` rides.
+     */
+    dataVolumeSizeGb: number;
     /**
      * Optional S3 cold-tier configuration.
      * When omitted, single-tier hot storage on the EC2 EBS volume is used.

package/dist/lib/resources/aws/database/clickhouseUserData.js

@@ -362,9 +362,14 @@ chmod 644 "$MOUNT_POINT/server-certs/client.xml"
 `
         : "";
     // The CWAgent-namespace metrics have no producer without this — no
-    // CloudWatch Agent is installed. disk_used_percent feeds the host disk
-    // alarms; mem_used_percent is dashboards-only (the CWAgent memory alarm is
-    // retired — memory alarming lives on ECS MemoryUtilization). The ASG name
+    // CloudWatch Agent is installed. disk_used_percent feeds the disk WARN
+    // alarm (trajectory) and disk_free_gib the disk CRITICAL alarm (imminent
+    // breakage — merges need free bytes, not free percent, so the paging signal
+    // is absolute and the warn one is not); mem_used_percent is dashboards-only
+    // (the CWAgent memory alarm is retired — memory alarming lives on ECS
+    // MemoryUtilization). Free space is read in MiB and divided rather than
+    // taken from `--block-size=1G`, whose integer truncation would quantise the
+    // metric to 1 GiB steps against a floor measured in tens. The ASG name
     // (the alarms' dimension) is unknowable at synth (LaunchTemplate → ASG is a
     // CFN cycle), so it resolves at run time: IMDS instance tags first, with a
     // describe-auto-scaling-instances fallback. AL2023 ECS AMIs ship no cron;
@@ -375,10 +380,10 @@ chmod 644 "$MOUNT_POINT/server-certs/client.xml"
     // timer never re-triggers a still-activating unit, so ONE wedged run (hung
     // IMDS or AWS call) would silence the publisher forever — every curl carries
     // --max-time and the unit a TimeoutStartSec under the 60s cadence. The disk
-    // metric publishes only when the data volume is actually mounted: `df` on
+    // metrics publish only when the data volume is actually mounted: `df` on
     // an unmounted path reports the ROOT filesystem, a false-healthy number
     // that would defeat the disk-critical alarm's missing-data=BREACHING flip.
-    const { namespace, memoryMetric, diskMetric, asgDimension } = CLICKHOUSE_HOST_METRICS;
+    const { namespace, memoryMetric, diskMetric, diskFreeMetric, asgDimension } = CLICKHOUSE_HOST_METRICS;
     const metricsPublisher = `
 cat > /usr/local/bin/fjall-ch-metrics.sh << 'METRICSEOF'
 #!/bin/bash
@@ -401,8 +406,10 @@ METRICS=("MetricName=${memoryMetric},Dimensions=[{Name=${asgDimension},Value=$AS
 if mountpoint -q ${CLICKHOUSE_DATA_MOUNT_PATH}; then
   DISK=$(df --output=pcent ${CLICKHOUSE_DATA_MOUNT_PATH} | tail -n 1 | tr -dc '0-9')
   METRICS+=("MetricName=${diskMetric},Dimensions=[{Name=${asgDimension},Value=$ASG}],Value=$DISK,Unit=Percent")
+  FREE=$(df --output=avail --block-size=1M ${CLICKHOUSE_DATA_MOUNT_PATH} | tail -n 1 | tr -dc '0-9' | awk '{printf "%.2f", $1/1024}')
+  METRICS+=("MetricName=${diskFreeMetric},Dimensions=[{Name=${asgDimension},Value=$ASG}],Value=$FREE,Unit=Gigabytes")
 else
-  echo "fjall-ch-metrics: ${CLICKHOUSE_DATA_MOUNT_PATH} not mounted, skipping disk metric" >&2
+  echo "fjall-ch-metrics: ${CLICKHOUSE_DATA_MOUNT_PATH} not mounted, skipping disk metrics" >&2
 fi
 aws cloudwatch put-metric-data --region "$REGION" --namespace ${namespace} --metric-data "\${METRICS[@]}"
 METRICSEOF
@@ -490,6 +497,15 @@ fi
 mkdir -p "$MOUNT_POINT"
 mount "$DEVICE" "$MOUNT_POINT"
 
+# The declared size below is load-bearing, and it MUST be a statement, not a
+# comment — compactUserDataScript strips comments from the emitted script.
+# User data is part of the launch template, so the size appearing verbatim in
+# the emitted script is what turns a storageGb change into a new template
+# version -> instance refresh -> the boot-time resize2fs below. Without it a
+# volume increase never relaunches anything and the grown capacity stays
+# unusable until an unrelated deploy replaces the instance.
+DECLARED_DATA_VOLUME_GIB=${options.dataVolumeSizeGb}
+#
 # Grow the filesystem to fill the volume. Raising the CDK volume size issues a
 # ModifyVolume against the block device only — ext4 keeps its old size until
 # resize2fs runs, so without this line every capacity increase is INERT and the

package/dist/lib/resources/aws/database/clickhouseXmlRenderer.js

@@ -1,15 +1,36 @@
 function camelToSnakeCase(input) {
     return input.replace(/[A-Z]/g, (m) => "_" + m.toLowerCase());
 }
+function renderConstraintsBlock(constraints) {
+    const settings = Object.entries(constraints).map(([key, constraint]) => {
+        const xmlName = camelToSnakeCase(key);
+        const children = [];
+        if (constraint.min !== undefined) {
+            children.push(`                    <min>${String(constraint.min)}</min>`);
+        }
+        if (constraint.max !== undefined) {
+            children.push(`                    <max>${String(constraint.max)}</max>`);
+        }
+        if (constraint.readonly === true) {
+            children.push(`                    <readonly/>`);
+        }
+        return `                <${xmlName}>\n${children.join("\n")}\n                </${xmlName}>`;
+    });
+    return `            <constraints>\n${settings.join("\n")}\n            </constraints>`;
+}
 function renderProfileSpec(spec) {
+    const { constraints, ...scalars } = spec;
     const lines = [];
-    for (const [key, value] of Object.entries(spec)) {
+    for (const [key, value] of Object.entries(scalars)) {
         if (value === undefined)
             continue;
         const xmlName = camelToSnakeCase(key);
         const xmlValue = typeof value === "boolean" ? (value ? "1" : "0") : String(value);
         lines.push(`            <${xmlName}>${xmlValue}</${xmlName}>`);
     }
+    if (constraints !== undefined && Object.keys(constraints).length > 0) {
+        lines.push(renderConstraintsBlock(constraints));
+    }
     return lines.join("\n");
 }
 function renderDefaultUserBlock(access) {

package/dist/lib/resources/aws/monitoring/clickhouseAlarms.d.ts

@@ -23,10 +23,39 @@ export interface ClickHouseAlarmThresholds {
      *  CWAgent host-memory alarm is retired (its threshold meant a different
      *  headroom per instance size; the timer metric remains for dashboards). */
     memoryThreshold?: number;
-    /** ClickHouse data volume disk % used. Default 70 (warn) — paired with critical at 85. */
+    /** ClickHouse data volume disk % used. Default 70. The TRAJECTORY signal —
+     *  "this volume is filling up, plan growth" — which is the one disk
+     *  question a percentage answers well at any volume size. The imminent-
+     *  breakage question is answered in GiB by `diskFreeCriticalGib`.
+     *
+     *  The nightly restore-verify transiently doubles the database's on-disk
+     *  footprint, so a database above ~35 % of the volume crosses a 70 % warn
+     *  during every verify window. That is the knob's honest reading — the
+     *  volume genuinely is that full while the scratch copy exists — but when
+     *  the nightly warn becomes noise, the cures are a larger `storageGb`, a
+     *  higher threshold here, or `backupVerify: false`; the critical floor is
+     *  protected from the same transient by the verify gate's ×2 margin. */
     diskWarnThreshold?: number;
-    /** ClickHouse data volume disk % used. Default 85. */
-    diskCriticalThreshold?: number;
+    /** Free space on the ClickHouse data volume, in GiB, below which the
+     *  critical alarm pages. Defaults to `defaultDiskFreeCriticalGib(storageGb)`
+     *  — the larger of an absolute floor and a share of the volume.
+     *
+     *  Absolute rather than a percentage BECAUSE the failure it guards is
+     *  absolute: a merge needs free bytes proportional to the parts it is
+     *  merging, not to the volume, and the server stops accepting inserts on a
+     *  fixed reserve. A percentage critical was either redundant (on a small
+     *  volume the free-space floor is crossed first) or noise (on a large one
+     *  85 % used still leaves hundreds of usable GiB) — never uniquely useful.
+     *
+     *  Must stay under a fifth of `storageGb`, so the floor is a slice of the
+     *  disk rather than most of it; `resolveClickHouseStorage` enforces the
+     *  pair together, since neither value is checkable alone.
+     *
+     *  Replaces `diskCriticalThreshold` (a percentage of the volume used). The
+     *  rename carries a unit change, so there is no value that means the same
+     *  thing in both — a declaration on the old key fails to compile rather
+     *  than deploying a threshold read as the wrong quantity. */
+    diskFreeCriticalGib?: number;
     /** `<Error>`/`<Fatal>` server-log lines per 5 min. Default 30. */
     serverErrorsPer5Min?: number;
     /** Failed `executeQuery:` error lines per 5 min. Default 10 — absolute, not
@@ -52,7 +81,7 @@ export interface ClickHouseAlarmThresholds {
  * backup heartbeat window. The remaining fields are consumed by the pattern's
  * service-alarm and logAlarms wiring (see `ClickHouseAlarmThresholds`).
  */
-export type ClickHouseHostAlarmThresholds = Pick<ClickHouseAlarmThresholds, "cpuThreshold" | "diskWarnThreshold" | "diskCriticalThreshold" | "backupHeartbeatWindowHours">;
+export type ClickHouseHostAlarmThresholds = Pick<ClickHouseAlarmThresholds, "cpuThreshold" | "diskWarnThreshold" | "backupHeartbeatWindowHours">;
 export declare const BACKUP_HEARTBEAT_DEFAULT_WINDOW_HOURS = 26;
 /**
  * Ceiling derivation — an ORDERING bound, not CloudWatch expressibility.
@@ -97,7 +126,25 @@ export interface ClickHouseAlarmsProps {
      * Omitted when `backupSchedule: false` — no backup task, no log group.
      */
     backupTaskLogGroup?: ILogGroup;
+    /**
+     * Whether the backup task runs the restore-verify (the pattern's
+     * `backupVerify` prop, already resolved). `false` withholds the two verify
+     * alarms so a deliberately disabled verify never leaves marker-counting
+     * alarms armed against a script that emits no markers — the alarm half and
+     * the script half of the contract disable together. The backup-failure
+     * alarm and the success heartbeat stay: they monitor the backup itself.
+     */
+    backupVerifyEnabled?: boolean;
     config?: ClickHouseHostAlarmThresholds;
+    /**
+     * Critical free-space floor in GiB, ALREADY RESOLVED. Not read off `config`
+     * because its default depends on the volume size and its legality depends
+     * on the volume size — both of which live on the pattern, not here. The
+     * pattern resolves the pair once (`resolveClickHouseStorage`) and hands the
+     * answer down; re-deriving it here would be a second source for a number
+     * that must agree with the volume it guards.
+     */
+    diskFreeCriticalGib: number;
     /** Application ID for webhook-to-application alarm mapping. */
     applicationId?: string;
 }
@@ -107,7 +154,9 @@ export interface ClickHouseAlarmsProps {
  * (never installed on these hosts) but by the lightweight put-metric-data
  * timer the user-data installs (see `buildClickHouseUserData`), which publishes
  * `mem_used_percent` / `disk_used_percent` under the same names and
- * `AutoScalingGroupName` dimension the agent would use — plus the
+ * `AutoScalingGroupName` dimension the agent would use, plus `disk_free_gib`,
+ * which has no CloudWatch Agent equivalent and exists because the disk
+ * question that pages must be answered in bytes rather than percent — plus the
  * backup-failure log alarm when a backup-task log group is supplied:
  *
  * - **Backup failures** — `AccessDenied` or `S3Exception` from the backup

package/dist/lib/resources/aws/monitoring/clickhouseAlarms.js

@@ -5,7 +5,7 @@ import { Metric } from "aws-cdk-lib/aws-cloudwatch";
 import { FilterPattern, MetricFilter } from "aws-cdk-lib/aws-logs";
 import { ALARM_DEFAULTS, registerAlarm, tagAlarmsWithApplicationId, buildAlarmDescription } from "./alarmDefaults.js";
 import { METRIC_NAMESPACE, stackScopedMetricNamespace } from "./metricNamespaces.js";
-import { CLICKHOUSE_HOST_METRICS } from "../database/clickhouseConstants.js";
+import { CLICKHOUSE_BACKUP_VERIFY_FAILED_MARKER, CLICKHOUSE_BACKUP_VERIFY_SKIPPED_MARKER, CLICKHOUSE_HOST_METRICS } from "../database/clickhouseConstants.js";
 export const BACKUP_HEARTBEAT_DEFAULT_WINDOW_HOURS = 26;
 /**
  * Ceiling derivation — an ORDERING bound, not CloudWatch expressibility.
@@ -55,7 +55,9 @@ export function validateClickHouseAlarmThresholds(config) {
  * (never installed on these hosts) but by the lightweight put-metric-data
  * timer the user-data installs (see `buildClickHouseUserData`), which publishes
  * `mem_used_percent` / `disk_used_percent` under the same names and
- * `AutoScalingGroupName` dimension the agent would use — plus the
+ * `AutoScalingGroupName` dimension the agent would use, plus `disk_free_gib`,
+ * which has no CloudWatch Agent equivalent and exists because the disk
+ * question that pages must be answered in bytes rather than percent — plus the
  * backup-failure log alarm when a backup-task log group is supplied:
  *
  * - **Backup failures** — `AccessDenied` or `S3Exception` from the backup
@@ -79,7 +81,7 @@ export function validateClickHouseAlarmThresholds(config) {
  * see (the EC2 host and the backup task).
  */
 export function createClickHouseAlarms(props) {
-    const { scope, instanceRole, asgName, alarmTopic, backupTaskLogGroup, config = {}, applicationId } = props;
+    const { scope, instanceRole, asgName, alarmTopic, backupTaskLogGroup, backupVerifyEnabled = true, config = {}, diskFreeCriticalGib, applicationId } = props;
     validateClickHouseAlarmThresholds(config);
     const alarms = [];
     const snsAction = new SnsAction(alarmTopic);
@@ -116,24 +118,28 @@ export function createClickHouseAlarms(props) {
         treatMissingData: TreatMissingData.NOT_BREACHING
     });
     registerAlarm(diskWarnAlarm, snsAction, alarms);
-    const diskCriticalThreshold = config.diskCriticalThreshold ?? 85;
     const diskCriticalAlarm = new Alarm(scope, "ClickHouseDiskCriticalAlarm", {
-        alarmDescription: buildAlarmDescription(`ClickHouse data volume above ${diskCriticalThreshold}% used — imminent insert failures`, applicationId),
+        alarmDescription: buildAlarmDescription(`ClickHouse data volume under ${diskFreeCriticalGib} GiB free — merges and inserts are about to fail`, applicationId),
         metric: new Metric({
             namespace: CLICKHOUSE_HOST_METRICS.namespace,
-            metricName: CLICKHOUSE_HOST_METRICS.diskMetric,
+            metricName: CLICKHOUSE_HOST_METRICS.diskFreeMetric,
             dimensionsMap: { [CLICKHOUSE_HOST_METRICS.asgDimension]: asgName },
             period: Duration.minutes(5),
-            statistic: "Average"
+            // Minimum, not Average: a single sample under the floor is the event,
+            // and averaging a dip against the surrounding minute hides exactly the
+            // moment a large merge claims the last of the disk.
+            statistic: "Minimum"
         }),
-        threshold: diskCriticalThreshold,
+        threshold: diskFreeCriticalGib,
         evaluationPeriods: 2,
         datapointsToAlarm: 2,
-        comparisonOperator: ComparisonOperator.GREATER_THAN_THRESHOLD,
+        comparisonOperator: ComparisonOperator.LESS_THAN_THRESHOLD,
         // BREACHING, unlike the other host alarms: a dead metrics publisher looks
         // identical to a full disk (no datapoints either way), and this alarm's
         // whole job is to fire before inserts start failing. The 2×5min window
-        // absorbs the boot gap on instance replacement.
+        // absorbs the boot gap on instance replacement — and, on the deploy that
+        // first introduces the metric, the instance refresh that starts
+        // publishing it. Expect one transient ALARM on that deploy.
         treatMissingData: TreatMissingData.BREACHING
     });
     registerAlarm(diskCriticalAlarm, snsAction, alarms);
@@ -163,6 +169,16 @@ export function createClickHouseAlarms(props) {
             treatMissingData: TreatMissingData.NOT_BREACHING
         });
         registerAlarm(backupFailureAlarm, snsAction, alarms);
+        if (backupVerifyEnabled) {
+            createBackupVerifyAlarms({
+                scope,
+                backupTaskLogGroup,
+                namespace: backupFailureNamespace,
+                snsAction,
+                applicationId,
+                alarms
+            });
+        }
         createBackupHeartbeat({
             scope,
             backupTaskLogGroup,
@@ -176,6 +192,86 @@ export function createClickHouseAlarms(props) {
     tagAlarmsWithApplicationId(alarms, applicationId);
     return alarms;
 }
+/**
+ * Restore-verify alarms — the half of the backup story `BACKUP_CREATED`
+ * cannot tell.
+ *
+ * The success marker the heartbeat reads means the BACKUP statement returned.
+ * It says nothing about whether the bytes can be read back, and the cheap
+ * check that looks like it would say so does not: a `structure_only` restore
+ * of a backup whose entire `data/` tree had been deleted returns `RESTORED`
+ * on 26.3.17.56, because it reads `metadata/` and never opens a part. So the
+ * backup task restores in full into a scratch database and emits its own
+ * verdict, and these two alarms read it.
+ *
+ * - **Verify failed** (`BACKUP_VERIFY_FAILED`, Sum >= 1 per hour) — the
+ *   backup that was just written could not be restored, or restored without
+ *   the tables the live database has, or restored empty. This is the alarm
+ *   that means the backups are not backups.
+ * - **Verify skipped for size** (`BACKUP_VERIFY_SKIPPED_FOR_SIZE`, Sum >= 1
+ *   per DAY) — the restore would not have left the critical free-space floor
+ *   clear, so the task declined to run it. Slower than its sibling because
+ *   it is not an emergency on the day it first fires; it is nonetheless a
+ *   page, because from that day on nothing is checking that the backups
+ *   restore, and a degradation nobody is told about is the failure mode this
+ *   whole mechanism exists to remove. The cure is a larger `storageGb` or
+ *   verification moved off-box.
+ *
+ * Both treat missing data as NOT_BREACHING: a run that never happened is the
+ * heartbeat's question, not this one's, and double-paging one outage from two
+ * alarms buys nothing.
+ */
+function createBackupVerifyAlarms(props) {
+    const { scope, backupTaskLogGroup, namespace, snsAction, applicationId, alarms } = props;
+    const verifyFailedMetricName = "ClickHouseBackupVerifyFailedCount";
+    new MetricFilter(scope, "ClickHouseBackupVerifyFailedMetricFilter", {
+        logGroup: backupTaskLogGroup,
+        metricNamespace: namespace,
+        metricName: verifyFailedMetricName,
+        filterPattern: FilterPattern.allTerms(CLICKHOUSE_BACKUP_VERIFY_FAILED_MARKER),
+        metricValue: "1",
+        defaultValue: 0
+    });
+    const verifyFailedAlarm = new Alarm(scope, "ClickHouseBackupVerifyFailedAlarm", {
+        alarmDescription: buildAlarmDescription("ClickHouse backup could not be restored — the backup completed but does not read back; treat the retained backups as unproven", applicationId),
+        metric: new Metric({
+            namespace,
+            metricName: verifyFailedMetricName,
+            period: Duration.hours(1),
+            statistic: "Sum"
+        }),
+        threshold: 1,
+        evaluationPeriods: 1,
+        datapointsToAlarm: 1,
+        comparisonOperator: ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
+        treatMissingData: TreatMissingData.NOT_BREACHING
+    });
+    registerAlarm(verifyFailedAlarm, snsAction, alarms);
+    const verifySkippedMetricName = "ClickHouseBackupVerifySkippedCount";
+    new MetricFilter(scope, "ClickHouseBackupVerifySkippedMetricFilter", {
+        logGroup: backupTaskLogGroup,
+        metricNamespace: namespace,
+        metricName: verifySkippedMetricName,
+        filterPattern: FilterPattern.allTerms(CLICKHOUSE_BACKUP_VERIFY_SKIPPED_MARKER),
+        metricValue: "1",
+        defaultValue: 0
+    });
+    const verifySkippedAlarm = new Alarm(scope, "ClickHouseBackupVerifySkippedAlarm", {
+        alarmDescription: buildAlarmDescription("ClickHouse backup restore-verify skipped — the database no longer fits a verify restore alongside itself; raise storageGb or move verification off-box", applicationId),
+        metric: new Metric({
+            namespace,
+            metricName: verifySkippedMetricName,
+            period: Duration.days(1),
+            statistic: "Sum"
+        }),
+        threshold: 1,
+        evaluationPeriods: 1,
+        datapointsToAlarm: 1,
+        comparisonOperator: ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
+        treatMissingData: TreatMissingData.NOT_BREACHING
+    });
+    registerAlarm(verifySkippedAlarm, snsAction, alarms);
+}
 /**
  * Backup success heartbeat — the complete-but-slow half of the backup alarm
  * pair (the AccessDenied/S3Exception failure alarm above is the fast half).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fjall/components-infrastructure",
-  "version": "16.0.0",
+  "version": "17.0.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/fjall-tech/fjall.git",
@@ -80,8 +80,8 @@
   },
   "dependencies": {
     "@aws-sdk/client-organizations": "^3.1098.0",
-    "@fjall/generator": "^16.0.0",
-    "@fjall/util": "^16.0.0",
+    "@fjall/generator": "^17.0.0",
+    "@fjall/util": "^17.0.0",
     "constructs": "^10.7.2"
   },
   "overrides": {