@fjall/components-infrastructure 16.0.1 → 18.0.0

package/dist/lib/resources/aws/monitoring/clickhouseAlarms.js

@@ -5,7 +5,7 @@ import { Metric } from "aws-cdk-lib/aws-cloudwatch";
 import { FilterPattern, MetricFilter } from "aws-cdk-lib/aws-logs";
 import { ALARM_DEFAULTS, registerAlarm, tagAlarmsWithApplicationId, buildAlarmDescription } from "./alarmDefaults.js";
 import { METRIC_NAMESPACE, stackScopedMetricNamespace } from "./metricNamespaces.js";
-import { CLICKHOUSE_HOST_METRICS } from "../database/clickhouseConstants.js";
+import { CLICKHOUSE_BACKUP_VERIFY_FAILED_MARKER, CLICKHOUSE_BACKUP_VERIFY_SKIPPED_MARKER, CLICKHOUSE_HOST_METRICS } from "../database/clickhouseConstants.js";
 export const BACKUP_HEARTBEAT_DEFAULT_WINDOW_HOURS = 26;
 /**
  * Ceiling derivation — an ORDERING bound, not CloudWatch expressibility.
@@ -55,7 +55,9 @@ export function validateClickHouseAlarmThresholds(config) {
  * (never installed on these hosts) but by the lightweight put-metric-data
  * timer the user-data installs (see `buildClickHouseUserData`), which publishes
  * `mem_used_percent` / `disk_used_percent` under the same names and
- * `AutoScalingGroupName` dimension the agent would use — plus the
+ * `AutoScalingGroupName` dimension the agent would use, plus `disk_free_gib`,
+ * which has no CloudWatch Agent equivalent and exists because the disk
+ * question that pages must be answered in bytes rather than percent — plus the
  * backup-failure log alarm when a backup-task log group is supplied:
  *
  * - **Backup failures** — `AccessDenied` or `S3Exception` from the backup
@@ -79,7 +81,7 @@ export function validateClickHouseAlarmThresholds(config) {
  * see (the EC2 host and the backup task).
  */
 export function createClickHouseAlarms(props) {
-    const { scope, instanceRole, asgName, alarmTopic, backupTaskLogGroup, config = {}, applicationId } = props;
+    const { scope, instanceRole, asgName, alarmTopic, backupTaskLogGroup, backupVerifyEnabled = true, config = {}, diskFreeCriticalGib, applicationId } = props;
     validateClickHouseAlarmThresholds(config);
     const alarms = [];
     const snsAction = new SnsAction(alarmTopic);
@@ -116,24 +118,28 @@ export function createClickHouseAlarms(props) {
         treatMissingData: TreatMissingData.NOT_BREACHING
     });
     registerAlarm(diskWarnAlarm, snsAction, alarms);
-    const diskCriticalThreshold = config.diskCriticalThreshold ?? 85;
     const diskCriticalAlarm = new Alarm(scope, "ClickHouseDiskCriticalAlarm", {
-        alarmDescription: buildAlarmDescription(`ClickHouse data volume above ${diskCriticalThreshold}% used — imminent insert failures`, applicationId),
+        alarmDescription: buildAlarmDescription(`ClickHouse data volume under ${diskFreeCriticalGib} GiB free — merges and inserts are about to fail`, applicationId),
         metric: new Metric({
             namespace: CLICKHOUSE_HOST_METRICS.namespace,
-            metricName: CLICKHOUSE_HOST_METRICS.diskMetric,
+            metricName: CLICKHOUSE_HOST_METRICS.diskFreeMetric,
             dimensionsMap: { [CLICKHOUSE_HOST_METRICS.asgDimension]: asgName },
             period: Duration.minutes(5),
-            statistic: "Average"
+            // Minimum, not Average: a single sample under the floor is the event,
+            // and averaging a dip against the surrounding minute hides exactly the
+            // moment a large merge claims the last of the disk.
+            statistic: "Minimum"
         }),
-        threshold: diskCriticalThreshold,
+        threshold: diskFreeCriticalGib,
         evaluationPeriods: 2,
         datapointsToAlarm: 2,
-        comparisonOperator: ComparisonOperator.GREATER_THAN_THRESHOLD,
+        comparisonOperator: ComparisonOperator.LESS_THAN_THRESHOLD,
         // BREACHING, unlike the other host alarms: a dead metrics publisher looks
         // identical to a full disk (no datapoints either way), and this alarm's
         // whole job is to fire before inserts start failing. The 2×5min window
-        // absorbs the boot gap on instance replacement.
+        // absorbs the boot gap on instance replacement — and, on the deploy that
+        // first introduces the metric, the instance refresh that starts
+        // publishing it. Expect one transient ALARM on that deploy.
         treatMissingData: TreatMissingData.BREACHING
     });
     registerAlarm(diskCriticalAlarm, snsAction, alarms);
@@ -163,6 +169,16 @@ export function createClickHouseAlarms(props) {
             treatMissingData: TreatMissingData.NOT_BREACHING
         });
         registerAlarm(backupFailureAlarm, snsAction, alarms);
+        if (backupVerifyEnabled) {
+            createBackupVerifyAlarms({
+                scope,
+                backupTaskLogGroup,
+                namespace: backupFailureNamespace,
+                snsAction,
+                applicationId,
+                alarms
+            });
+        }
         createBackupHeartbeat({
             scope,
             backupTaskLogGroup,
@@ -176,6 +192,86 @@ export function createClickHouseAlarms(props) {
     tagAlarmsWithApplicationId(alarms, applicationId);
     return alarms;
 }
+/**
+ * Restore-verify alarms — the half of the backup story `BACKUP_CREATED`
+ * cannot tell.
+ *
+ * The success marker the heartbeat reads means the BACKUP statement returned.
+ * It says nothing about whether the bytes can be read back, and the cheap
+ * check that looks like it would say so does not: a `structure_only` restore
+ * of a backup whose entire `data/` tree had been deleted returns `RESTORED`
+ * on 26.3.17.56, because it reads `metadata/` and never opens a part. So the
+ * backup task restores in full into a scratch database and emits its own
+ * verdict, and these two alarms read it.
+ *
+ * - **Verify failed** (`BACKUP_VERIFY_FAILED`, Sum >= 1 per hour) — the
+ *   backup that was just written could not be restored, or restored without
+ *   the tables the live database has, or restored empty. This is the alarm
+ *   that means the backups are not backups.
+ * - **Verify skipped for size** (`BACKUP_VERIFY_SKIPPED_FOR_SIZE`, Sum >= 1
+ *   per DAY) — the restore would not have left the critical free-space floor
+ *   clear, so the task declined to run it. Slower than its sibling because
+ *   it is not an emergency on the day it first fires; it is nonetheless a
+ *   page, because from that day on nothing is checking that the backups
+ *   restore, and a degradation nobody is told about is the failure mode this
+ *   whole mechanism exists to remove. The cure is a larger `storageGb` or
+ *   verification moved off-box.
+ *
+ * Both treat missing data as NOT_BREACHING: a run that never happened is the
+ * heartbeat's question, not this one's, and double-paging one outage from two
+ * alarms buys nothing.
+ */
+function createBackupVerifyAlarms(props) {
+    const { scope, backupTaskLogGroup, namespace, snsAction, applicationId, alarms } = props;
+    const verifyFailedMetricName = "ClickHouseBackupVerifyFailedCount";
+    new MetricFilter(scope, "ClickHouseBackupVerifyFailedMetricFilter", {
+        logGroup: backupTaskLogGroup,
+        metricNamespace: namespace,
+        metricName: verifyFailedMetricName,
+        filterPattern: FilterPattern.allTerms(CLICKHOUSE_BACKUP_VERIFY_FAILED_MARKER),
+        metricValue: "1",
+        defaultValue: 0
+    });
+    const verifyFailedAlarm = new Alarm(scope, "ClickHouseBackupVerifyFailedAlarm", {
+        alarmDescription: buildAlarmDescription("ClickHouse backup could not be restored — the backup completed but does not read back; treat the retained backups as unproven", applicationId),
+        metric: new Metric({
+            namespace,
+            metricName: verifyFailedMetricName,
+            period: Duration.hours(1),
+            statistic: "Sum"
+        }),
+        threshold: 1,
+        evaluationPeriods: 1,
+        datapointsToAlarm: 1,
+        comparisonOperator: ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
+        treatMissingData: TreatMissingData.NOT_BREACHING
+    });
+    registerAlarm(verifyFailedAlarm, snsAction, alarms);
+    const verifySkippedMetricName = "ClickHouseBackupVerifySkippedCount";
+    new MetricFilter(scope, "ClickHouseBackupVerifySkippedMetricFilter", {
+        logGroup: backupTaskLogGroup,
+        metricNamespace: namespace,
+        metricName: verifySkippedMetricName,
+        filterPattern: FilterPattern.allTerms(CLICKHOUSE_BACKUP_VERIFY_SKIPPED_MARKER),
+        metricValue: "1",
+        defaultValue: 0
+    });
+    const verifySkippedAlarm = new Alarm(scope, "ClickHouseBackupVerifySkippedAlarm", {
+        alarmDescription: buildAlarmDescription("ClickHouse backup restore-verify skipped — the database no longer fits a verify restore alongside itself; raise storageGb or move verification off-box", applicationId),
+        metric: new Metric({
+            namespace,
+            metricName: verifySkippedMetricName,
+            period: Duration.days(1),
+            statistic: "Sum"
+        }),
+        threshold: 1,
+        evaluationPeriods: 1,
+        datapointsToAlarm: 1,
+        comparisonOperator: ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
+        treatMissingData: TreatMissingData.NOT_BREACHING
+    });
+    registerAlarm(verifySkippedAlarm, snsAction, alarms);
+}
 /**
  * Backup success heartbeat — the complete-but-slow half of the backup alarm
  * pair (the AccessDenied/S3Exception failure alarm above is the fast half).

package/dist/lib/resources/aws/storage/s3.d.ts

@@ -37,15 +37,27 @@ export interface S3BucketProps extends BucketProps {
     publicReadAccess?: boolean;
     websiteHosting?: WebsiteHostingConfig;
     /**
-     * Name a `fjallBucket(<name>)` alias target uses to import this bucket's
-     * website exports. Defaults to `bucketName ?? id`, which is right for a
-     * direct `new S3Bucket(scope, "Assets", …)` — `id` is then the author's own
-     * handle. A wrapper that nests the bucket under a derived id MUST pass this:
-     * `Storage` builds its bucket at `${id}Bucket`, so without it a
-     * `fjallBucket("assets")` record would import `AssetsWebsiteEndpoint`
-     * against an exported `AssetsBucketWebsiteEndpoint`.
+     * Name a `fjallBucket(<app>, <name>)` alias target uses to import this
+     * bucket's website exports. Defaults to `bucketName ?? id`, which is right
+     * for a direct `new S3Bucket(scope, "Assets", …)` — `id` is then the
+     * author's own handle. A wrapper that nests the bucket under a derived id
+     * MUST pass this: `Storage` builds its bucket at `${id}Bucket`, so without
+     * it a `fjallBucket(app, "assets")` record would import
+     * `…-Assets-WebsiteEndpoint` against an exported
+     * `…-AssetsBucket-WebsiteEndpoint`.
      */
     aliasTargetName?: string;
+    /**
+     * The owning Fjall app's name — the other half of the website-export key
+     * (`bucketWebsiteEndpointExportName(appName, bucket)`). REQUIRED whenever
+     * `websiteHosting` is set: export names are account+region-global and every
+     * scaffolded app names its website bucket identically, so an unqualified
+     * export makes two website-bearing apps mutually undeployable. The
+     * `StorageFactory` path fills this from the App automatically (as
+     * `CdnFactory` does for `Cdn`); only direct `new S3Bucket`/`new Storage`
+     * construction needs to pass it.
+     */
+    appName?: string;
     /**
      * Declarative bucket-policy statements, each appended via
      * `addToResourcePolicy`. The TLS-only `enforceSSL` deny is always applied

package/dist/lib/resources/aws/storage/s3.js

@@ -18,7 +18,7 @@ function toResourcePolicyPrincipal(identifier) {
 export class S3Bucket extends Bucket {
     backupVaultTier;
     constructor(scope, id, props = {}) {
-        const { websiteHosting, backupVaultTier, resourcePolicyStatements, aliasTargetName, ...cdkProps } = props;
+        const { websiteHosting, backupVaultTier, resourcePolicyStatements, aliasTargetName, appName, ...cdkProps } = props;
         const isPublic = props.publicReadAccess === true || websiteHosting !== undefined;
         const versioned = props.versioned ?? shouldAutoVersion(backupVaultTier);
         const removalPolicy = props.removalPolicy ?? toRemovalPolicy(envAwareRemovalPolicyDefault());
@@ -74,6 +74,18 @@ export class S3Bucket extends Bucket {
             Tags.of(this).add(SDK_PRE_EMPTY_TAG_KEY, "true");
         }
         if (websiteHosting) {
+            if (appName === undefined) {
+                // Refusing beats falling back to a bucket-only export name: the
+                // consumer (`resolveBucketTarget`) always imports the app-qualified
+                // pair, so an unqualified export could never be resolved — it would
+                // only surface at CloudFormation execution as "No export named …",
+                // or worse, collide with a sibling app's identically-named bucket.
+                throw new Error(`S3 bucket '${id}': websiteHosting requires 'appName' — the ` +
+                    "website exports are keyed on (app, bucket) so two apps' " +
+                    "identically-named buckets stay deployable side by side. " +
+                    "App-managed paths (StorageFactory/CdnFactory) fill it " +
+                    "automatically; pass appName explicitly for direct construction.");
+            }
             const targetBucketName = aliasTargetName ?? props.bucketName ?? id;
             // Only the export name routes through the contract helper. `safeBucket`
             // stays local so the CfnOutput logical ids are unchanged — unifying the
@@ -81,7 +93,7 @@ export class S3Bucket extends Bucket {
             const safeBucket = toPascalCase(targetBucketName.replace(/[^A-Za-z0-9-]/g, ""));
             new CfnOutput(this, `${safeBucket}WebsiteEndpoint`, {
                 key: `${safeBucket}WebsiteEndpoint`,
-                exportName: bucketWebsiteEndpointExportName(targetBucketName),
+                exportName: bucketWebsiteEndpointExportName(appName, targetBucketName),
                 value: this.bucketWebsiteDomainName,
                 description: `S3 website endpoint for ${id} (consumed by Domain alias targets)`
             });
@@ -93,7 +105,7 @@ export class S3Bucket extends Bucket {
             }
             new CfnOutput(this, `${safeBucket}WebsiteHostedZoneId`, {
                 key: `${safeBucket}WebsiteHostedZoneId`,
-                exportName: bucketWebsiteHostedZoneIdExportName(targetBucketName),
+                exportName: bucketWebsiteHostedZoneIdExportName(appName, targetBucketName),
                 value: websiteHostedZoneId,
                 description: `Route 53 hosted zone id for the S3 website endpoint of ${id}`
             });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fjall/components-infrastructure",
-  "version": "16.0.1",
+  "version": "18.0.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/fjall-tech/fjall.git",
@@ -80,8 +80,8 @@
   },
   "dependencies": {
     "@aws-sdk/client-organizations": "^3.1098.0",
-    "@fjall/generator": "^16.0.1",
-    "@fjall/util": "^16.0.1",
+    "@fjall/generator": "^18.0.0",
+    "@fjall/util": "^18.0.0",
     "constructs": "^10.7.2"
   },
   "overrides": {