@fjall/components-infrastructure 19.0.0 → 21.0.0

package/dist/lib/patterns/aws/computeEcs.js

@@ -26,7 +26,7 @@ import { FjallLogger } from "../../utils/validationLogger.js";
 import { VALIDATION_PATTERNS } from "@fjall/generator";
 import { evaluateBakeGuard } from "@fjall/util/docker";
 import { toKebab, buildParameterPath } from "@fjall/util";
-import { SCHEMA_GATE_CH_CA_CERT_ENV, SCHEMA_GATE_CH_DATABASE_ENV, SCHEMA_GATE_CH_URL_ENV, SCHEMA_GATE_CONTAINER_NAME, SCHEMA_GATE_DB_URL_BASE_ENV, SCHEMA_GATE_ECR_REPO_NAME } from "@fjall/util/migration";
+import { SCHEMA_GATE_CH_CA_CERT_ENV, SCHEMA_GATE_CH_DATABASE_ENV, SCHEMA_GATE_CH_URL_ENV, SCHEMA_GATE_CONTAINER_NAME, SCHEMA_GATE_DB_URL_BASE_ENV, SCHEMA_GATE_ECR_REPO_NAME, SCHEMA_GATE_START_TIMEOUT_SECONDS } from "@fjall/util/migration";
 import { CONSTRUCTS_VERSION } from "../../utils/engineCompat.js";
 import { validateEc2ServiceSizing, validateEc2TaskMemoryFit, validateEcsDomainConfig, validateSecretName } from "../../resources/aws/compute/ecsValidation.js";
 import { validateSharedEc2CapacityConfig } from "../../resources/aws/compute/ecsCapacityConfig.js";
@@ -271,14 +271,6 @@ export function validateEcsProps(props) {
     }
 }
 const DEFAULT_MIGRATE_CONTAINER_NAME = "migrate";
-/**
- * How long dependent containers wait for the gate to reach SUCCESS before
- * giving up (stopping the task pre-RUNNING). ECS declares this on the
- * depended-ON container. 120s is the verified Fargate ceiling
- * (design 2026-08-10 § 4); the runner's worst-case retry budget
- * (3 connects × 10s + 2 sleeps × 5s, PG and CH in parallel) fits inside it.
- */
-const SCHEMA_GATE_START_TIMEOUT_SECONDS = 120;
 /**
  * Postgres-family scheme test for the gate's v1 eligibility filter. The URL
  * base is `getConnectionString()` output whose host may be an unresolved CDK
@@ -732,15 +724,13 @@ function assertMigrationSecretsResolvable(service, effectiveMigrations) {
  *                      container gains a `dependsOn` SUCCESS edge on it.
  * @internal Exported for testing only
  */
-export function buildContainerConfigs(service, schemaVersionEnv, annotationsScope, chGate, schemaGate) {
+export function buildContainerConfigs(service, schemaVersionEnv, annotationsScope, chSchemaVersionEnv, schemaGate) {
     const userContainers = service.containers && service.containers.length > 0
         ? service.containers
         : undefined;
     const expanded = service.migrations
         ? expandMigrationsSugar(service, userContainers)
         : userContainers;
-    const chSchemaVersionEnv = chGate?.environment;
-    const chGateSecretsImport = chGate?.secretsImport;
     const mergeSchemaEnv = (authored) => mergePgSchemaVersionEnv({
         authored,
         resolved: schemaVersionEnv,
@@ -755,18 +745,6 @@ export function buildContainerConfigs(service, schemaVersionEnv, annotationsScop
         annotationsScope,
         overrideRemedy: "set `schemaGate: false` to silence this warning."
     });
-    const mergeChGateSecretsImport = (authored) => {
-        if (chGateSecretsImport === undefined)
-            return authored;
-        if (Object.keys(chGateSecretsImport).length === 0)
-            return authored;
-        const merged = { ...(authored ?? {}) };
-        for (const [k, v] of Object.entries(chGateSecretsImport)) {
-            if (merged[k] === undefined)
-                merged[k] = v;
-        }
-        return merged;
-    };
     const mergeAllSchemaEnv = (authored) => mergeChSchemaEnv(mergeSchemaEnv(authored));
     // Prepend the synthetic gate container and wire every other container to
     // wait on its SUCCESS. In init-container mode the gate itself waits on the
@@ -817,7 +795,7 @@ export function buildContainerConfigs(service, schemaVersionEnv, annotationsScop
                 port: c.port,
                 environment: mergeAllSchemaEnv(c.environment),
                 secrets: c.secrets,
-                secretsImport: mergeChGateSecretsImport(c.secretsImport),
+                secretsImport: c.secretsImport,
                 command: c.command,
                 entryPoint: c.entryPoint,
                 essential: c.essential,
@@ -830,14 +808,10 @@ export function buildContainerConfigs(service, schemaVersionEnv, annotationsScop
         }));
     }
     const fallbackEnv = mergeAllSchemaEnv(undefined);
-    const fallbackSecretsImport = mergeChGateSecretsImport(undefined);
     return wireSchemaGate([
         {
             name: `${service.name}Container`,
-            ...(fallbackEnv !== undefined && { environment: fallbackEnv }),
-            ...(fallbackSecretsImport !== undefined && {
-                secretsImport: fallbackSecretsImport
-            })
+            ...(fallbackEnv !== undefined && { environment: fallbackEnv })
         }
     ]);
 }
@@ -919,12 +893,12 @@ export class EcsCompute extends Construct {
         this.schemaGateImageTags = schemaGateImageTags;
         const services = props.services.map((service) => {
             const schemaVersionEnv = this.resolveSchemaVersionEnv(service);
-            const chGate = this.resolveClickHouseSchemaVersionEnv(service);
-            const schemaGateSpec = this.resolveSchemaGateMaterialisation(service, schemaVersionEnv, chGate);
+            const chSchemaVersionEnv = this.resolveClickHouseSchemaVersionEnv(service);
+            const schemaGateSpec = this.resolveSchemaGateMaterialisation(service, schemaVersionEnv, chSchemaVersionEnv);
             if (schemaGateSpec?.defaultImageTag !== undefined) {
                 schemaGateImageTags.set(service.name, schemaGateSpec.defaultImageTag);
             }
-            const containers = buildContainerConfigs(service, schemaVersionEnv, this, chGate, schemaGateSpec);
+            const containers = buildContainerConfigs(service, schemaVersionEnv, this, chSchemaVersionEnv, schemaGateSpec);
             const { scalingType, minCapacity, maxCapacity, queueScaling } = resolveScalingConfig(service.scaling);
             const cloudMapService = service.serviceDiscovery !== undefined
                 ? App.getInstance().registerService({
@@ -1041,17 +1015,16 @@ export class EcsCompute extends Construct {
     }
     /**
      * ClickHouse mirror of `resolveSchemaVersionEnv`. Returns the
-     * `EXPECTED_CH_SCHEMA_VERSION` env entry alongside the schema-admin
-     * credentials the boot gate needs to authenticate against
-     * `_schema_migrations`, or `undefined` when the service is not gated.
+     * `EXPECTED_CH_SCHEMA_VERSION` env entry, or `undefined` when the service
+     * is not gated. Env only: app containers receive no ClickHouse credentials
+     * from the gate — the materialised gate container authenticates as the
+     * read-only gate identity via `getSchemaGateContribution()`, composed in
+     * `resolveSchemaGateMaterialisation`.
      *
      * - `schemaGate: false` → returns `undefined` (auditable opt-out — same flag
      *   covers BOTH PG and CH gates)
      * - No migrated CH in connections → returns `undefined`
-     * - Exactly one migrated CH → returns env `EXPECTED_CH_SCHEMA_VERSION` +
-     *   `SCHEMA_ADMIN_USER`, plus `secretsImport.SCHEMA_ADMIN_PASSWORD`. The
-     *   creds piggyback on the same gate resolution so adding the gate to a
-     *   service can't ship without the creds it needs to use it.
+     * - Exactly one migrated CH → returns `{ EXPECTED_CH_SCHEMA_VERSION }`
      * - Two or more migrated CHs → throws via `resolveClickHouseDatabaseForService`
      */
     resolveClickHouseSchemaVersionEnv(service) {
@@ -1063,16 +1036,7 @@ export class EcsCompute extends Construct {
         const version = db.getExpectedSchemaVersion();
         if (version === undefined)
             return undefined;
-        const environment = {
-            [EXPECTED_CH_SCHEMA_VERSION_ENV]: version
-        };
-        const secretsImport = {};
-        const gateContribution = db.getSchemaGateContribution();
-        if (gateContribution !== undefined) {
-            Object.assign(environment, gateContribution.environment);
-            Object.assign(secretsImport, gateContribution.secretsImport);
-        }
-        return { environment, secretsImport };
+        return { [EXPECTED_CH_SCHEMA_VERSION_ENV]: version };
     }
     /**
      * Decide whether the synthetic `fjall-schema-gate` container materialises
@@ -1095,17 +1059,19 @@ export class EcsCompute extends Construct {
      * - The default image URI cannot be derived because the constructs package
      *   version is unresolvable and no `schemaGate.image` override is set
      *
-     * The relational half re-resolves the migrated DB for its
-     * `getSchemaGateContribution()` (credential-free URL base + user/password
-     * secret imports). The CH half composes `CLICKHOUSE_URL` / database name /
-     * CA-cert import from the resolved CH database — `chGate` alone carries
-     * only the version + schema-admin entries the app containers get.
+     * Both halves re-resolve the migrated DB for its
+     * `getSchemaGateContribution()`: the relational half for the
+     * credential-free URL base + user/password secret imports, the CH half for
+     * the read-only gate identity (`FJALL_SCHEMA_GATE_CH_USER` +
+     * `FJALL_SCHEMA_GATE_CH_PASSWORD`) composed with `CLICKHOUSE_URL` /
+     * database name / CA-cert import — `chSchemaVersionEnv` alone carries only
+     * the version entry the app containers get.
      */
-    resolveSchemaGateMaterialisation(service, schemaVersionEnv, chGate) {
+    resolveSchemaGateMaterialisation(service, schemaVersionEnv, chSchemaVersionEnv) {
         const normalised = normaliseSchemaGate(service.schemaGate);
         if (!normalised.materialise)
             return undefined;
-        if (schemaVersionEnv === undefined && chGate === undefined) {
+        if (schemaVersionEnv === undefined && chSchemaVersionEnv === undefined) {
             return undefined;
         }
         if (service.migrations?.mode === "post-deploy") {
@@ -1154,14 +1120,15 @@ export class EcsCompute extends Construct {
                 eligibleHalves += 1;
             }
         }
-        if (chGate !== undefined) {
+        if (chSchemaVersionEnv !== undefined) {
             const db = resolveClickHouseDatabaseForService(service);
             if (db !== undefined) {
-                Object.assign(environment, chGate.environment, {
+                const contribution = db.getSchemaGateContribution();
+                Object.assign(environment, chSchemaVersionEnv, contribution?.environment, {
                     [SCHEMA_GATE_CH_URL_ENV]: db.getUrl(),
                     [SCHEMA_GATE_CH_DATABASE_ENV]: db.getDatabaseName()
                 });
-                Object.assign(secretsImport, chGate.secretsImport);
+                Object.assign(secretsImport, contribution?.secretsImport);
                 const caCert = db.getTlsCaCertImport();
                 if (caCert !== undefined) {
                     secretsImport[SCHEMA_GATE_CH_CA_CERT_ENV] = caCert;
@@ -1285,7 +1252,7 @@ export class EcsCompute extends Construct {
             schemaGate: entry.schemaGate
         };
         const schemaVersionEnv = this.resolveSchemaVersionEnv(gateSubject);
-        const chGate = this.resolveClickHouseSchemaVersionEnv(gateSubject);
+        const chSchemaVersionEnv = this.resolveClickHouseSchemaVersionEnv(gateSubject);
         const environment = mergeChSchemaVersionEnv({
             authored: mergePgSchemaVersionEnv({
                 authored: entry.environment,
@@ -1294,23 +1261,12 @@ export class EcsCompute extends Construct {
                 annotationsScope: this,
                 materialisedGateNote: false
             }),
-            resolved: chGate?.environment,
+            resolved: chSchemaVersionEnv,
             subject,
             annotationsScope: this,
             overrideRemedy: "remove the env entry to accept the resolved value."
         });
-        let secrets = entry.secrets;
-        if (chGate !== undefined && Object.keys(chGate.secretsImport).length > 0) {
-            const merged = {};
-            for (const [key, secretImport] of Object.entries(chGate.secretsImport)) {
-                if (entry.secrets?.[key] !== undefined)
-                    continue;
-                const secret = resolveImportedSecret(this, `${id}${toPascalCase(entry.name)}${key}Secret`, secretImport);
-                merged[key] = EcsSecret.fromSecretsManager(secret, secretImport.field);
-            }
-            secrets = { ...merged, ...(entry.secrets ?? {}) };
-        }
-        return { environment, secrets };
+        return { environment, secrets: entry.secrets };
     }
     /**
      * A sibling service that consumes a hook-migrated database materialises a

package/dist/lib/patterns/aws/interfaces/database.d.ts

@@ -262,8 +262,9 @@ export interface IClickHouseDatabase extends IDatabase, IConnectable, IMigration
     getUser(name: string): Secret;
     /**
      * Returns the Fjall `Secret` wrapper for the named user, or `undefined`
-     * when the user is not declared. Use for optional secret contributions
-     * (e.g. `SCHEMA_ADMIN_PASSWORD` only when a `schema_admin` user exists).
+     * when the user is not declared in `schemaAdmin:` / `managedPasswords:`.
+     * Framework identities (`fjall_schema_gate`, `fjall_maintenance`) are not
+     * reachable here — their secrets are wired by the construct itself.
      */
     tryGetUser(name: string): Secret | undefined;
     /**
@@ -320,19 +321,19 @@ export interface IClickHouseDatabase extends IDatabase, IConnectable, IMigration
      */
     getExpectedSchemaVersion(): string | undefined;
     /**
-     * Schema-admin credentials for the boot-time schema-version gate run by
-     * every container that connects to this database. Returns the username
-     * (plaintext env) and the password-secret import (Secrets-Manager-backed
-     * env) so callers can authenticate against `_schema_migrations`. Returns
-     * `undefined` when no `migrations:` config is declared on this database —
-     * the gate is the only consumer, so without migrations there's nothing
-     * for it to verify.
+     * ClickHouse credentials for the materialised boot-time schema gate
+     * (`fjall-schema-gate` container). The gate authenticates as the read-only
+     * gate identity (`fjall_schema_gate`), never as the schema admin: env
+     * `FJALL_SCHEMA_GATE_CH_USER` carries the fixed name and secret import
+     * `FJALL_SCHEMA_GATE_CH_PASSWORD` its password. Returns `undefined` when
+     * no `migrations:` config is declared on this database — the gate is the
+     * only consumer, so without migrations there's nothing for it to verify.
      *
      * Distinct from `getMigrationContributions()`: that's the per-migration
-     * task contract (env + secrets + IAM + egress for running migrations);
-     * this is the narrower per-runtime-service contract (env + secret only)
-     * needed by the boot gate. Both contributions name the same secret —
-     * synth-time merging is the caller's concern.
+     * task contract (env + secrets + IAM + egress for running migrations as
+     * the schema admin); this is the narrower gate-container contract (env +
+     * secret only). App containers receive neither — they get the
+     * `EXPECTED_CH_SCHEMA_VERSION` env entry alone.
      */
     getSchemaGateContribution(): {
         readonly environment: Record<string, string>;

package/dist/lib/resources/aws/compute/ecs.js

@@ -315,7 +315,8 @@ export default class EcsCluster extends Construct {
                 targetGroup,
                 config: typeof serviceProps.alarms === "object" ? serviceProps.alarms : {},
                 alarmTopic: this.props.alertsTopic,
-                applicationId: this.props.applicationId
+                applicationId: this.props.applicationId,
+                containerInsightsEnabled: this.props.containerInsights === true
             });
             if (serviceProps.logAlarms && serviceProps.logAlarms.length > 0) {
                 createLogPatternAlarms({

package/dist/lib/resources/aws/compute/ecsTypes.d.ts

@@ -751,9 +751,11 @@ export interface EcsClusterProps {
     /**
      * CloudWatch Container Insights (default off). Enabling it bills
      * ~$0.30/metric-month of custom `ECS/ContainerInsights` metrics per cluster
-     * plus performance-log ingestion; nothing in the constructs consumes that
-     * namespace (alarms sit on `AWS/ECS` and `Fjall/ECS`), so it is an explicit
-     * opt-in for clusters whose operators read the per-task console breakdown.
+     * plus performance-log ingestion. The per-service running-tasks alarm is
+     * the one construct consumer of that namespace (`RunningTaskCount` exists
+     * nowhere else) and is skipped when insights are off; every other alarm
+     * sits on `AWS/ECS` and `Fjall/ECS`. Explicit opt-in for clusters whose
+     * operators want the running-tasks alarm or the per-task console breakdown.
      */
     containerInsights?: true;
 }

package/dist/lib/resources/aws/database/clickhouseBackupScript.d.ts

@@ -20,6 +20,8 @@
 export interface ClickHouseBackupScriptParams {
     /** Fully-formed `clickhouse-client …` invocation, minus `--query`. */
     readonly client: string;
+    /** The migration principal the activity pre-flight defers to. */
+    readonly schemaAdminName: string;
     /** Database the backup covers, and the restore compares against. */
     readonly databaseName: string;
     /** `https://<bucket>.s3.<region>.amazonaws.com/backup/` — trailing slash. */
@@ -99,5 +101,15 @@ export interface ClickHouseBackupScriptParams {
  * sets `set -eu`, but ONLY when TLS is active — so a script depending on
  * `set -e` would abort on the first failure with TLS on and blunder past it
  * with TLS off. Explicit branches behave identically either way.
+ *
+ * Statements go through the maintenance runners (`clickhouseMaintenanceScript`)
+ * rather than the bare client, so a `Code: 202` from the maintenance
+ * identity's concurrency cap is waited out instead of failing the run.
+ * Which runner is deliberate per statement: `fjall_ch_run` for the scratch
+ * drop and the BACKUP itself, whose failures have no marker of their own;
+ * `fjall_ch_try` for every statement whose failure the script already
+ * reports (the restore's FAILED branch, the SKIPPED probe branch) or
+ * tolerates (the row/table reads, the trailing drop) — a second failed-status
+ * marker there would page the same event twice.
  */
 export declare function buildClickHouseBackupScript(params: ClickHouseBackupScriptParams): string;

package/dist/lib/resources/aws/database/clickhouseBackupScript.js

@@ -18,7 +18,9 @@
  * behaviour the alarm is built on.
  */
 import { CLICKHOUSE_BACKUP_SCRATCH_DATABASE, CLICKHOUSE_BACKUP_VERIFY_FREE_SPACE_MARGIN, CLICKHOUSE_BACKUP_VERIFY_FAILED_MARKER, CLICKHOUSE_BACKUP_VERIFY_OK_MARKER, CLICKHOUSE_BACKUP_VERIFY_SKIPPED_MARKER, clickHouseBackupVerifyTimeoutSeconds } from "./clickhouseConstants.js";
+import { buildMaintenancePreamble, maintenanceOkMarker, maintenanceRun, maintenanceTry } from "./clickhouseMaintenanceScript.js";
 const BYTES_PER_GIB = 1024 ** 3;
+const JOB = "backup";
 /**
  * Builds the backup + restore-verify script.
  *
@@ -83,23 +85,39 @@ const BYTES_PER_GIB = 1024 ** 3;
  * sets `set -eu`, but ONLY when TLS is active — so a script depending on
  * `set -e` would abort on the first failure with TLS on and blunder past it
  * with TLS off. Explicit branches behave identically either way.
+ *
+ * Statements go through the maintenance runners (`clickhouseMaintenanceScript`)
+ * rather than the bare client, so a `Code: 202` from the maintenance
+ * identity's concurrency cap is waited out instead of failing the run.
+ * Which runner is deliberate per statement: `fjall_ch_run` for the scratch
+ * drop and the BACKUP itself, whose failures have no marker of their own;
+ * `fjall_ch_try` for every statement whose failure the script already
+ * reports (the restore's FAILED branch, the SKIPPED probe branch) or
+ * tolerates (the row/table reads, the trailing drop) — a second failed-status
+ * marker there would page the same event twice.
  */
 export function buildClickHouseBackupScript(params) {
-    const { client, databaseName, backupDestUrl, diskFreeCriticalGib, storageGb, verify } = params;
+    const { client, schemaAdminName, databaseName, backupDestUrl, diskFreeCriticalGib, storageGb, verify } = params;
+    const preamble = buildMaintenancePreamble({
+        client,
+        job: JOB,
+        schemaAdminName
+    });
     const backupStatements = [
         "STAMP=$(date +%Y%m%d-%H%M%S)",
         `DEST="${backupDestUrl}backup-$STAMP/"`,
-        `${client} --query "BACKUP DATABASE ${databaseName} TO S3('$DEST')" || exit 1`
+        `${maintenanceRun("backup", `BACKUP DATABASE ${databaseName} TO S3('$DEST')`)} || exit 1`
     ];
     if (!verify) {
-        return backupStatements.join("; ");
+        return `${preamble}\n${[...backupStatements, maintenanceOkMarker(JOB)].join("; ")}`;
     }
     const scratch = CLICKHOUSE_BACKUP_SCRATCH_DATABASE;
     const requiredFreeBytes = Math.round(diskFreeCriticalGib *
         BYTES_PER_GIB *
         CLICKHOUSE_BACKUP_VERIFY_FREE_SPACE_MARGIN);
     const verifyTimeoutSeconds = clickHouseBackupVerifyTimeoutSeconds(storageGb);
-    const dropScratch = `${client} --query "DROP DATABASE IF EXISTS ${scratch} SYNC"`;
+    const dropScratchSql = `DROP DATABASE IF EXISTS ${scratch} SYNC`;
+    const dropScratchTolerated = maintenanceTry("drop_scratch", dropScratchSql);
     const sourceRowsQuery = `SELECT sum(rows) FROM system.parts WHERE database = '${databaseName}' AND active`;
     const fitVerdictQuery = `SELECT if((SELECT free_space FROM system.disks WHERE name = 'default')` +
         ` - (SELECT sum(bytes_on_disk) FROM system.parts WHERE database = '${databaseName}' AND active)` +
@@ -107,17 +125,19 @@ export function buildClickHouseBackupScript(params) {
     const missingTablesQuery = `SELECT count() FROM (SELECT name FROM system.tables WHERE database = '${databaseName}' AND name NOT LIKE '.inner%'` +
         ` EXCEPT SELECT name FROM system.tables WHERE database = '${scratch}' AND name NOT LIKE '.inner%')`;
     const restoredRowsQuery = `SELECT sum(rows) FROM system.parts WHERE database = '${scratch}' AND active`;
-    return [
-        `${dropScratch} || exit 1`,
-        `SRCROWS=$(${client} --query "${sourceRowsQuery}") || SRCROWS=unknown`,
+    const body = [
+        `${maintenanceRun("drop_scratch", dropScratchSql)} || exit 1`,
+        `SRCROWS=$(${maintenanceTry("source_rows", sourceRowsQuery)}) || SRCROWS=unknown`,
         ...backupStatements,
-        `FIT=$(${client} --query "${fitVerdictQuery}") || { echo "${CLICKHOUSE_BACKUP_VERIFY_SKIPPED_MARKER} $DEST free-space probe failed — verify did not run"; exit 1; }`,
-        `if [ "$FIT" != "FITS" ]; then echo "${CLICKHOUSE_BACKUP_VERIFY_SKIPPED_MARKER} $DEST restore would not leave ${requiredFreeBytes} bytes free"; exit 0; fi`,
-        `timeout ${verifyTimeoutSeconds} ${client} --query "RESTORE DATABASE ${databaseName} AS ${scratch} FROM S3('$DEST')" || { echo "${CLICKHOUSE_BACKUP_VERIFY_FAILED_MARKER} $DEST restore raised or exceeded ${verifyTimeoutSeconds}s"; ${dropScratch} || true; exit 1; }`,
-        `MISSING=$(${client} --query "${missingTablesQuery}") || MISSING=unknown`,
-        `ROWS=$(${client} --query "${restoredRowsQuery}") || ROWS=0`,
-        `${dropScratch} || true`,
+        `FIT=$(${maintenanceTry("free_space_probe", fitVerdictQuery)}) || { echo "${CLICKHOUSE_BACKUP_VERIFY_SKIPPED_MARKER} $DEST free-space probe failed — verify did not run"; exit 1; }`,
+        `if [ "$FIT" != "FITS" ]; then echo "${CLICKHOUSE_BACKUP_VERIFY_SKIPPED_MARKER} $DEST restore would not leave ${requiredFreeBytes} bytes free"; ${maintenanceOkMarker(JOB)}; exit 0; fi`,
+        `${maintenanceTry("restore", `RESTORE DATABASE ${databaseName} AS ${scratch} FROM S3('$DEST')`, verifyTimeoutSeconds)} || { echo "${CLICKHOUSE_BACKUP_VERIFY_FAILED_MARKER} $DEST restore raised or exceeded ${verifyTimeoutSeconds}s"; ${dropScratchTolerated} || true; exit 1; }`,
+        `MISSING=$(${maintenanceTry("missing_tables", missingTablesQuery)}) || MISSING=unknown`,
+        `ROWS=$(${maintenanceTry("restored_rows", restoredRowsQuery)}) || ROWS=0`,
+        `${dropScratchTolerated} || true`,
         `if [ "$MISSING" != "0" ] || { [ "$ROWS" = "0" ] && [ "$SRCROWS" != "0" ]; }; then echo "${CLICKHOUSE_BACKUP_VERIFY_FAILED_MARKER} $DEST missing_tables=$MISSING restored_rows=$ROWS source_rows=$SRCROWS"; exit 1; fi`,
-        `echo "${CLICKHOUSE_BACKUP_VERIFY_OK_MARKER} $DEST restored_rows=$ROWS source_rows=$SRCROWS"`
+        `echo "${CLICKHOUSE_BACKUP_VERIFY_OK_MARKER} $DEST restored_rows=$ROWS source_rows=$SRCROWS"`,
+        maintenanceOkMarker(JOB)
     ].join("; ");
+    return `${preamble}\n${body}`;
 }

package/dist/lib/resources/aws/database/clickhouseConstants.d.ts

@@ -303,8 +303,36 @@ export declare const BACKUP_TASK_CPU_UNITS = 256;
 export declare const BACKUP_RETENTION_DAYS = 14;
 /** Scratch database the backup task restores into to prove the backup it just
  *  wrote can be read back. Dropped on the way in and on the way out; never
- *  queried by the application. */
-export declare const CLICKHOUSE_BACKUP_SCRATCH_DATABASE = "fjall_backup_verify";
+ *  queried by the application. Canonical source is `@fjall/util/migration`
+ *  because `@fjall/clickhouse` grants `fjall_maintenance` DROP on exactly
+ *  this database — coupled values per `.claude/rules/code-quality.md §
+ *  "Coupled values: shared source at 2 occurrences"`. */
+export { CLICKHOUSE_BACKUP_SCRATCH_DATABASE } from "@fjall/util/migration";
+/** The two scheduled maintenance jobs that authenticate as
+ *  `fjall_maintenance`. Drives the per-job log-group → metric-filter wiring
+ *  in `createClickHouseAlarms` and the `job=` field of the status markers. */
+export declare const CLICKHOUSE_MAINTENANCE_JOBS: readonly ["optimise", "backup"];
+export type ClickHouseMaintenanceJob = (typeof CLICKHOUSE_MAINTENANCE_JOBS)[number];
+/** Total wall-clock budget a maintenance statement may spend waiting out
+ *  `Code: 202` (TOO_MANY_SIMULTANEOUS_QUERIES) from the maintenance
+ *  identity's `max_concurrent_queries_for_user = 1` cap before the job
+ *  reports `status=failed reason=busy`. Ten minutes: long enough to outlast
+ *  the sibling sidecar's per-statement gaps, short enough that a wedged
+ *  holder pages inside the hour the failed-status alarm samples. */
+export declare const MAINTENANCE_BUSY_BUDGET_SECONDS = 600;
+/** Prefix of the maintenance sidecars' status log markers. The CloudWatch
+ *  metric filters match `${prefix}${status}` as a whole token, so the
+ *  three statuses must remain distinct full words after the `=`. */
+export declare const MAINTENANCE_STATUS_MARKER_PREFIX = "fjall:maintenance:status=";
+export declare const MAINTENANCE_STATUS: {
+    readonly ok: "ok";
+    readonly deferred: "deferred";
+    readonly failed: "failed";
+};
+export type MaintenanceStatus = (typeof MAINTENANCE_STATUS)[keyof typeof MAINTENANCE_STATUS];
+/** The log token a maintenance status marker line starts with — shared by
+ *  the script builders (emit) and the alarm factory (match). */
+export declare function maintenanceStatusMarker(status: MaintenanceStatus): string;
 /** Multiple of the critical free-space floor the verify restore must leave
  *  clear to proceed.
  *

package/dist/lib/resources/aws/database/clickhouseConstants.js

@@ -345,8 +345,36 @@ export const BACKUP_TASK_CPU_UNITS = 256;
 export const BACKUP_RETENTION_DAYS = 14;
 /** Scratch database the backup task restores into to prove the backup it just
  *  wrote can be read back. Dropped on the way in and on the way out; never
- *  queried by the application. */
-export const CLICKHOUSE_BACKUP_SCRATCH_DATABASE = "fjall_backup_verify";
+ *  queried by the application. Canonical source is `@fjall/util/migration`
+ *  because `@fjall/clickhouse` grants `fjall_maintenance` DROP on exactly
+ *  this database — coupled values per `.claude/rules/code-quality.md §
+ *  "Coupled values: shared source at 2 occurrences"`. */
+export { CLICKHOUSE_BACKUP_SCRATCH_DATABASE } from "@fjall/util/migration";
+/** The two scheduled maintenance jobs that authenticate as
+ *  `fjall_maintenance`. Drives the per-job log-group → metric-filter wiring
+ *  in `createClickHouseAlarms` and the `job=` field of the status markers. */
+export const CLICKHOUSE_MAINTENANCE_JOBS = ["optimise", "backup"];
+/** Total wall-clock budget a maintenance statement may spend waiting out
+ *  `Code: 202` (TOO_MANY_SIMULTANEOUS_QUERIES) from the maintenance
+ *  identity's `max_concurrent_queries_for_user = 1` cap before the job
+ *  reports `status=failed reason=busy`. Ten minutes: long enough to outlast
+ *  the sibling sidecar's per-statement gaps, short enough that a wedged
+ *  holder pages inside the hour the failed-status alarm samples. */
+export const MAINTENANCE_BUSY_BUDGET_SECONDS = 600;
+/** Prefix of the maintenance sidecars' status log markers. The CloudWatch
+ *  metric filters match `${prefix}${status}` as a whole token, so the
+ *  three statuses must remain distinct full words after the `=`. */
+export const MAINTENANCE_STATUS_MARKER_PREFIX = "fjall:maintenance:status=";
+export const MAINTENANCE_STATUS = {
+    ok: "ok",
+    deferred: "deferred",
+    failed: "failed"
+};
+/** The log token a maintenance status marker line starts with — shared by
+ *  the script builders (emit) and the alarm factory (match). */
+export function maintenanceStatusMarker(status) {
+    return `${MAINTENANCE_STATUS_MARKER_PREFIX}${status}`;
+}
 /** Multiple of the critical free-space floor the verify restore must leave
  *  clear to proceed.
  *

package/dist/lib/resources/aws/database/clickhouseMaintenanceScript.d.ts

@@ -0,0 +1,69 @@
+/**
+ * Shell fragments shared by the two maintenance sidecars (OPTIMIZE FINAL and
+ * BACKUP), which both authenticate as `fjall_maintenance` under a
+ * `max_concurrent_queries_for_user = 1` cap.
+ *
+ * Two things the cap forces on every statement the sidecars issue:
+ *
+ * 1. **The migrate ↔ maintenance mutex.** A migration holds the DDL slot for
+ *    minutes; a maintenance job that started underneath it would either
+ *    contend for parts the migration is rewriting or — worse — make the
+ *    migration's own probes lose the slot. So each job opens with an activity
+ *    pre-flight (`buildControlPlaneActivityQuery` against the schema admin)
+ *    and exits `deferred` (0, a metric but no page) when a migration is under
+ *    way. The runner asks the mirror question about `fjall_maintenance`.
+ *
+ * 2. **`Code: 202` is a wait, not a failure.** With a cap of one, the second
+ *    query from the same identity is refused with TOO_MANY_SIMULTANEOUS_
+ *    QUERIES. That is the sibling sidecar (or this job's own previous
+ *    statement still winding down), so the statement runner sleeps with
+ *    bounded backoff and retries until `MAINTENANCE_BUSY_BUDGET_SECONDS` is
+ *    spent — only then is it a failure, and it says `reason=busy` so the
+ *    responder knows to look for the holder rather than the statement.
+ *
+ * Every marker goes to stderr. The backup script captures several statement
+ * results with `$(…)`, and a marker on stdout would land in the captured
+ * value; ECS awslogs carries both streams to the same log group, which is
+ * where the metric filters read them.
+ *
+ * `sh` only (the ClickHouse image's `/bin/sh` is dash): no `$RANDOM`, no
+ * arrays, no `pipefail`. Jitter comes from the nanosecond clock instead —
+ * `%s%N` rather than `%N`, because a bare `%N` is zero-padded and a leading
+ * zero makes dash read the number as octal.
+ *
+ * The TLS preamble these fragments follow sets `set -eu` ONLY when TLS is
+ * active, so nothing here relies on `-e`: every failure path is an explicit
+ * `||` / `if` branch and the function bodies use `… && return 0` shapes that
+ * behave identically with or without it.
+ */
+import { type ClickHouseMaintenanceJob } from "./clickhouseConstants.js";
+/** Runs a statement; on failure emits `status=failed reason=<label>` (or
+ *  `reason=busy label=<label>` when the busy budget ran out) and returns 1.
+ *  For statements whose failure the job treats as fatal. */
+export declare const MAINTENANCE_RUN_FN = "fjall_ch_run";
+/** Runs a statement with the same `Code: 202` retry loop but emits NO
+ *  failed-status marker on a non-busy failure — the caller owns that branch
+ *  (a tolerated read, or a statement with its own marker such as the backup
+ *  restore). The busy-exhausted marker is still emitted: a statement that
+ *  never got a slot is always a job failure. */
+export declare const MAINTENANCE_TRY_FN = "fjall_ch_try";
+export interface MaintenancePreambleParams {
+    /** Fully-formed `clickhouse-client …` invocation, minus `--query`. */
+    readonly client: string;
+    readonly job: ClickHouseMaintenanceJob;
+    /** The migration principal — the other side of the mutex. */
+    readonly schemaAdminName: string;
+}
+/**
+ * Newline-joined preamble: sets `FJALL_JOB`, defines the two statement
+ * runners, then runs the activity pre-flight. Exits the script itself on
+ * `deferred` (0) or a probe failure (1), so the job body that follows only
+ * ever runs when no migration is active.
+ */
+export declare function buildMaintenancePreamble(params: MaintenancePreambleParams): string;
+/** `fjall_ch_run <label> "<sql>" [timeoutSeconds]` — see `MAINTENANCE_RUN_FN`. */
+export declare function maintenanceRun(label: string, sql: string, timeoutSeconds?: number): string;
+/** `fjall_ch_try <label> "<sql>" [timeoutSeconds]` — see `MAINTENANCE_TRY_FN`. */
+export declare function maintenanceTry(label: string, sql: string, timeoutSeconds?: number): string;
+/** The job's terminal success line. */
+export declare function maintenanceOkMarker(job: ClickHouseMaintenanceJob): string;