@fjall/components-infrastructure 39.0.3 → 39.0.4

package/dist/lib/patterns/aws/clickhouseDatabase.js

@@ -17,7 +17,8 @@ import { Secret } from "../../resources/aws/secrets/secret.js";
 import { LogGroup } from "../../resources/aws/logging/logGroup.js";
 import { StackEncryptionKey } from "../../resources/aws/secrets/stackEncryptionKey.js";
 import { createClickHouseSecurityGroup } from "../../resources/aws/database/clickhouseSecurityGroup.js";
-import { buildMaintenancePreamble, maintenanceOkMarker, maintenanceRun } from "../../resources/aws/database/clickhouseMaintenanceScript.js";
+import { buildMaintenancePreamble } from "../../resources/aws/database/clickhouseMaintenanceScript.js";
+import { buildClickHouseOptimiseScript } from "../../resources/aws/database/clickhouseOptimiseScript.js";
 import { buildClickHouseEntrypointWrapper, buildClickHouseUserData, generateUsersConfigXml } from "../../resources/aws/database/clickhouseUserData.js";
 import { toPascalCase } from "../../utils/capitaliseString.js";
 import { resolveAlertsTopic } from "../../utils/resolveAlertsTopic.js";
@@ -25,7 +26,7 @@ import { createClickHouseAlarms, validateClickHouseAlarmThresholds, BACKUP_HEART
 import { ClickHouseSchemaAdminSchema, ManagedPasswordNameSchema, ProfileSpecSchema, PROFILE_NAME_PATTERN } from "../../resources/aws/database/clickhouseSchemas.js";
 import { deriveClickHouseDefaultProfiles } from "../../resources/aws/database/clickhouseTuning.js";
 import { inferAmiHardwareType } from "../../resources/aws/compute/ecsConstants.js";
-import { CLICKHOUSE_DATABASE_NAME, DEFAULT_CLICKHOUSE_INSTANCE_TYPE, CLICKHOUSE_IMAGE, clickHouseTaskMemoryMiB, CLICKHOUSE_HTTP_PORT, CLICKHOUSE_HTTPS_PORT, CLICKHOUSE_NATIVE_PORT, CLICKHOUSE_TCP_SECURE_PORT, CLICKHOUSE_TLS_CERT_MOUNT_PATH, CLICKHOUSE_PROMETHEUS_PORT, CLICKHOUSE_DATA_MOUNT_PATH, CLICKHOUSE_SECRET_OPTIONS, CLICKHOUSE_SERVER_ROLE_TAG, CLICKHOUSE_HOST_METRICS, clickHouseUserSecretName, CLICKHOUSE_HEALTH_CHECK, CLICKHOUSE_STOP_TIMEOUT_SECONDS, CLICKHOUSE_EBS_DEVICE_NAME, CLICKHOUSE_CONFIG_SUBDIR, CLICKHOUSE_USERS_SUBDIR, userPasswordEnvName, OPTIMISE_FINAL_SCHEDULE, REPLACING_MERGE_TREE_TABLES, OPTIMISE_MV_TABLES, CLICKHOUSE_CLOUDMAP_SERVICE_NAME, CLICKHOUSE_SERVER_CONTAINER_NAME, CLICKHOUSE_SERVICE_NAME, OPTIMISE_TASK_MEMORY_MIB, OPTIMISE_TASK_CPU_UNITS, BACKUP_SCHEDULE, BACKUP_TASK_MEMORY_MIB, BACKUP_TASK_CPU_UNITS, BACKUP_RETENTION_DAYS } from "../../resources/aws/database/clickhouseConstants.js";
+import { CLICKHOUSE_DATABASE_NAME, DEFAULT_CLICKHOUSE_INSTANCE_TYPE, CLICKHOUSE_IMAGE, clickHouseTaskMemoryMiB, CLICKHOUSE_HTTP_PORT, CLICKHOUSE_HTTPS_PORT, CLICKHOUSE_NATIVE_PORT, CLICKHOUSE_TCP_SECURE_PORT, CLICKHOUSE_TLS_CERT_MOUNT_PATH, CLICKHOUSE_PROMETHEUS_PORT, CLICKHOUSE_DATA_MOUNT_PATH, CLICKHOUSE_SECRET_OPTIONS, CLICKHOUSE_SERVER_ROLE_TAG, CLICKHOUSE_HOST_METRICS, clickHouseUserSecretName, CLICKHOUSE_HEALTH_CHECK, CLICKHOUSE_STOP_TIMEOUT_SECONDS, CLICKHOUSE_EBS_DEVICE_NAME, CLICKHOUSE_CONFIG_SUBDIR, CLICKHOUSE_USERS_SUBDIR, userPasswordEnvName, OPTIMISE_FINAL_SCHEDULE, REPLACING_MERGE_TREE_TABLES, OPTIMISE_MV_TABLES, CLICKHOUSE_CLOUDMAP_SERVICE_NAME, CLICKHOUSE_SERVER_CONTAINER_NAME, CLICKHOUSE_SERVICE_NAME, OPTIMISE_TASK_MEMORY_MIB, OPTIMISE_TASK_CPU_UNITS, OPTIMISE_STATEMENT_RECEIVE_TIMEOUT_SECONDS, clickHouseBackupVerifyTimeoutSeconds, BACKUP_SCHEDULE, BACKUP_TASK_MEMORY_MIB, BACKUP_TASK_CPU_UNITS, BACKUP_RETENTION_DAYS } from "../../resources/aws/database/clickhouseConstants.js";
 import { CLICKHOUSE_MAX_BACKUP_RETENTION_DAYS, CLICKHOUSE_MIN_BACKUP_RETENTION_DAYS } from "@fjall/util/clickhouse";
 import { resolveClickHouseStorage, resolveClickHouseVolumePerformance } from "../../resources/aws/database/clickhouseStorage.js";
 import { buildClickHouseBackupScript } from "../../resources/aws/database/clickhouseBackupScript.js";
@@ -468,16 +469,6 @@ export class ClickHouseDatabase extends Construct {
         const dataAz = Stack.of(this).availabilityZones[0];
         const clickHouseHost = this.getHostEndpoint();
         const backupDestUrl = `https://${backupBucket.bucketName}.s3.${Stack.of(this).region}.amazonaws.com/backup/`;
-        const optimiseStatements = [
-            ...REPLACING_MERGE_TREE_TABLES.map((table) => ({
-                table,
-                sql: `OPTIMIZE TABLE ${CLICKHOUSE_DATABASE_NAME}.${table} FINAL`
-            })),
-            ...OPTIMISE_MV_TABLES.map((table) => ({
-                table,
-                sql: `OPTIMIZE TABLE ${CLICKHOUSE_DATABASE_NAME}.${table}`
-            }))
-        ];
         const adminSecret = expectDefined(userSecrets.get(schemaAdmin.name), `schemaAdmin '${schemaAdmin.name}' secret not minted.`);
         const maintenanceSecret = expectDefined(frameworkSecrets.get(FJALL_MAINTENANCE_USER), `framework identity '${FJALL_MAINTENANCE_USER}' secret not minted.`);
         const sidecarTlsPreamble = tlsActive
@@ -496,16 +487,19 @@ export class ClickHouseDatabase extends Construct {
         // own status marker and the task still exits non-zero so they stay
         // visible in the task history. Both scheduled tasks authenticate as the
         // maintenance identity; the password arrives via CLICKHOUSE_PASSWORD
-        // rather than argv.
+        // rather than argv. Each job's client names its own `receive_timeout`,
+        // because the client's 300 s default reports a statement the server is
+        // still finishing as failed.
         const clickHouseClient = `clickhouse-client --host ${clickHouseHost} --port ${nativePort} --user ${FJALL_MAINTENANCE_USER}${sidecarTlsClientArgs}`;
-        const optimiseScript = [
-            "FAILED=0",
-            ...optimiseStatements.map(({ table, sql }) => `${maintenanceRun(`optimise_${table}`, sql)} || FAILED=1`),
-            `if [ "$FAILED" = 0 ]; then ${maintenanceOkMarker("optimise")}; fi`,
-            'exit "$FAILED"'
-        ].join("; ");
+        const optimiseClient = `${clickHouseClient} --receive_timeout=${OPTIMISE_STATEMENT_RECEIVE_TIMEOUT_SECONDS}`;
+        const backupClient = `${clickHouseClient} --receive_timeout=${clickHouseBackupVerifyTimeoutSeconds(storageGb)}`;
+        const optimiseScript = buildClickHouseOptimiseScript({
+            databaseName: CLICKHOUSE_DATABASE_NAME,
+            replacingTables: REPLACING_MERGE_TREE_TABLES,
+            mvTables: OPTIMISE_MV_TABLES
+        });
         const optimisePreamble = buildMaintenancePreamble({
-            client: clickHouseClient,
+            client: optimiseClient,
             job: "optimise",
             schemaAdminName: schemaAdmin.name
         });
@@ -547,7 +541,7 @@ export class ClickHouseDatabase extends Construct {
                     "-c",
                     // Password via CLICKHOUSE_PASSWORD env, not --password on argv (argv → /proc/<pid>/cmdline).
                     `${sidecarTlsPreamble}${buildClickHouseBackupScript({
-                        client: clickHouseClient,
+                        client: backupClient,
                         schemaAdminName: schemaAdmin.name,
                         databaseName: CLICKHOUSE_DATABASE_NAME,
                         backupDestUrl,

package/dist/lib/resources/aws/database/clickhouseBackupScript.js

@@ -17,7 +17,7 @@
  * case loudly (`STD_EXCEPTION`, missing `checksums.txt`), which is the
  * behaviour the alarm is built on.
  */
-import { CLICKHOUSE_BACKUP_SCRATCH_DATABASE, CLICKHOUSE_BACKUP_VERIFY_FREE_SPACE_MARGIN, CLICKHOUSE_BACKUP_VERIFY_FAILED_MARKER, CLICKHOUSE_BACKUP_VERIFY_OK_MARKER, CLICKHOUSE_BACKUP_VERIFY_SKIPPED_MARKER, clickHouseBackupVerifyTimeoutSeconds } from "./clickhouseConstants.js";
+import { CLICKHOUSE_BACKUP_SCRATCH_DATABASE, CLICKHOUSE_BACKUP_VERIFY_FREE_SPACE_MARGIN, CLICKHOUSE_BACKUP_VERIFY_FAILED_MARKER, CLICKHOUSE_BACKUP_VERIFY_OK_MARKER, CLICKHOUSE_BACKUP_VERIFY_SKIPPED_MARKER, CLICKHOUSE_DEFAULT_DISK_NAME, clickHouseBackupVerifyTimeoutSeconds } from "./clickhouseConstants.js";
 import { buildMaintenancePreamble, maintenanceOkMarker, maintenanceRun, maintenanceTry } from "./clickhouseMaintenanceScript.js";
 const BYTES_PER_GIB = 1024 ** 3;
 const JOB = "backup";
@@ -119,7 +119,7 @@ export function buildClickHouseBackupScript(params) {
     const dropScratchSql = `DROP DATABASE IF EXISTS ${scratch} SYNC`;
     const dropScratchTolerated = maintenanceTry("drop_scratch", dropScratchSql);
     const sourceRowsQuery = `SELECT sum(rows) FROM system.parts WHERE database = '${databaseName}' AND active`;
-    const fitVerdictQuery = `SELECT if((SELECT free_space FROM system.disks WHERE name = 'default')` +
+    const fitVerdictQuery = `SELECT if((SELECT free_space FROM system.disks WHERE name = '${CLICKHOUSE_DEFAULT_DISK_NAME}')` +
         ` - (SELECT sum(bytes_on_disk) FROM system.parts WHERE database = '${databaseName}' AND active)` +
         ` >= ${requiredFreeBytes}, 'FITS', 'SKIP')`;
     const missingTablesQuery = `SELECT count() FROM (SELECT name FROM system.tables WHERE database = '${databaseName}' AND name NOT LIKE '.inner%'` +

package/dist/lib/resources/aws/database/clickhouseConstants.d.ts

@@ -253,14 +253,42 @@ export declare const CLICKHOUSE_HEALTH_CHECK: {
  *  m7g.medium under merge load risks `max_suspicious_broken_parts` on the
  *  next start. ECS upper bound is 120s. */
 export declare const CLICKHOUSE_STOP_TIMEOUT_SECONDS = 90;
-/** OPTIMIZE TABLE FINAL schedule.
- *  RMT tables carry min_age_to_force_merge_seconds=600 so the engine already merges
- *  old parts within 10 min; this task is a safety net for MVs (no engine-level setting)
- *  and for ReplacingMergeTree dedup under skewed write patterns. 6 hours is sufficient. */
+/** Optimise task schedule.
+ *  The ReplacingMergeTree tables carry `min_age_to_force_merge_seconds=600`
+ *  with `min_age_to_force_merge_on_partition_only=1`, so the engine force-merges
+ *  a partition whose newest part is ten minutes old, but only when the merge
+ *  selector has nothing else to do and the pool has a slot free. This task is
+ *  the backstop for that, and for the materialised views in
+ *  `OPTIMISE_MV_TABLES`. 6 hours is sufficient. */
 export declare const OPTIMISE_FINAL_SCHEDULE = "rate(6 hours)";
-/** Tables requiring periodic OPTIMIZE FINAL (ReplacingMergeTree only).
- *  Keep in sync with REPLACING_MERGE_TREE_TABLES in
- *  webapp/app/.server/lib/clickhouse/tenantQuery/tableConstants.ts (auto-FINAL). */
+/** ClickHouse's built-in name for the local data disk: the hot volume of
+ *  every storage policy, and the disk the free-space checks read. The server
+ *  defines it from `<path>`, so it is ClickHouse's name, not a choice made
+ *  here. The optimise sweep reads it to tell a partition wholly on local disk
+ *  from one that has moved to cold storage. */
+export declare const CLICKHOUSE_DEFAULT_DISK_NAME = "default";
+/** How long the optimise client waits on one statement before it reports it
+ *  failed, passed as `--receive_timeout`. It is the longest single-partition
+ *  merge the job accepts, and a statement that runs past it is reported
+ *  failed.
+ *
+ *  It bounds the client's wait, not the server's work. Measured on
+ *  production, clickhouse-server 26.3.17.56, 2026-09-30: at the client's
+ *  300 s default, every whole-table `log_events` merge was reported failed
+ *  (FJL-609), while the server's `system.query_log` recorded each one
+ *  finishing with code 0, after 724 to 1277 s. Over 13 days of
+ *  `system.part_log`, the longest single-partition merge of any listed table
+ *  was 225 s, on weekly `log_events` partitions of up to 3.8 GiB, so this is
+ *  about five times the worst observed. */
+export declare const OPTIMISE_STATEMENT_RECEIVE_TIMEOUT_SECONDS = 1200;
+/** ReplacingMergeTree tables the optimise task settles, partition by
+ *  partition (`buildClickHouseOptimiseScript`).
+ *
+ *  Coupled with the webapp, whose tenant table manifest marks the same tables
+ *  `replacingMergeTree`; a webapp test compares the two lists, reading this
+ *  one from the installed package. The list follows the engine, not what the
+ *  webapp reads with FINAL: a table that collapses duplicates on read still
+ *  needs its merges. */
 export declare const REPLACING_MERGE_TREE_TABLES: readonly ["application_metrics", "cost_records", "log_events", "log_fingerprints", "insights", "asset_inventory"];
 /** Subdirectory on the EBS volume for server config files (must match CDK volume mount). */
 export declare const CLICKHOUSE_CONFIG_SUBDIR = "server-config.d";

package/dist/lib/resources/aws/database/clickhouseConstants.js

@@ -279,14 +279,42 @@ export const CLICKHOUSE_HEALTH_CHECK = {
  *  m7g.medium under merge load risks `max_suspicious_broken_parts` on the
  *  next start. ECS upper bound is 120s. */
 export const CLICKHOUSE_STOP_TIMEOUT_SECONDS = 90;
-/** OPTIMIZE TABLE FINAL schedule.
- *  RMT tables carry min_age_to_force_merge_seconds=600 so the engine already merges
- *  old parts within 10 min; this task is a safety net for MVs (no engine-level setting)
- *  and for ReplacingMergeTree dedup under skewed write patterns. 6 hours is sufficient. */
+/** Optimise task schedule.
+ *  The ReplacingMergeTree tables carry `min_age_to_force_merge_seconds=600`
+ *  with `min_age_to_force_merge_on_partition_only=1`, so the engine force-merges
+ *  a partition whose newest part is ten minutes old, but only when the merge
+ *  selector has nothing else to do and the pool has a slot free. This task is
+ *  the backstop for that, and for the materialised views in
+ *  `OPTIMISE_MV_TABLES`. 6 hours is sufficient. */
 export const OPTIMISE_FINAL_SCHEDULE = "rate(6 hours)";
-/** Tables requiring periodic OPTIMIZE FINAL (ReplacingMergeTree only).
- *  Keep in sync with REPLACING_MERGE_TREE_TABLES in
- *  webapp/app/.server/lib/clickhouse/tenantQuery/tableConstants.ts (auto-FINAL). */
+/** ClickHouse's built-in name for the local data disk: the hot volume of
+ *  every storage policy, and the disk the free-space checks read. The server
+ *  defines it from `<path>`, so it is ClickHouse's name, not a choice made
+ *  here. The optimise sweep reads it to tell a partition wholly on local disk
+ *  from one that has moved to cold storage. */
+export const CLICKHOUSE_DEFAULT_DISK_NAME = "default";
+/** How long the optimise client waits on one statement before it reports it
+ *  failed, passed as `--receive_timeout`. It is the longest single-partition
+ *  merge the job accepts, and a statement that runs past it is reported
+ *  failed.
+ *
+ *  It bounds the client's wait, not the server's work. Measured on
+ *  production, clickhouse-server 26.3.17.56, 2026-09-30: at the client's
+ *  300 s default, every whole-table `log_events` merge was reported failed
+ *  (FJL-609), while the server's `system.query_log` recorded each one
+ *  finishing with code 0, after 724 to 1277 s. Over 13 days of
+ *  `system.part_log`, the longest single-partition merge of any listed table
+ *  was 225 s, on weekly `log_events` partitions of up to 3.8 GiB, so this is
+ *  about five times the worst observed. */
+export const OPTIMISE_STATEMENT_RECEIVE_TIMEOUT_SECONDS = 1200;
+/** ReplacingMergeTree tables the optimise task settles, partition by
+ *  partition (`buildClickHouseOptimiseScript`).
+ *
+ *  Coupled with the webapp, whose tenant table manifest marks the same tables
+ *  `replacingMergeTree`; a webapp test compares the two lists, reading this
+ *  one from the installed package. The list follows the engine, not what the
+ *  webapp reads with FINAL: a table that collapses duplicates on read still
+ *  needs its merges. */
 export const REPLACING_MERGE_TREE_TABLES = [
     "application_metrics",
     "cost_records",

package/dist/lib/resources/aws/database/clickhouseOptimiseScript.d.ts

@@ -0,0 +1,96 @@
+/**
+ * The optimise task's shell body: settle, partition by partition, the
+ * ReplacingMergeTree partitions that have something to settle, then compact
+ * the materialised views.
+ *
+ * A whole-table `OPTIMIZE TABLE t FINAL` is the wrong unit of work. These four
+ * reasons were measured on clickhouse-server 26.3.17.56, the image
+ * `CLICKHOUSE_IMAGE` pinned on 2026-09-30:
+ *
+ * 1. **It rewrites what is already settled.** FINAL merges every partition,
+ *    a partition that is already a single part included, so the work grows
+ *    with the table rather than with what changed.
+ * 2. **It rewrites cold data.** A merged part lands on the coldest volume its
+ *    sources sit on, so every cold partition is read back from object
+ *    storage and written to it again on every run.
+ * 3. **It stops silently.** At the first partition it cannot select it
+ *    returns success without touching the rest of the table.
+ * 4. **Its duration is the table's, not the change's.** The statement holds
+ *    the maintenance identity's single query slot for as long as the whole
+ *    table takes. On production, `clickhouse-client` printed its
+ *    `receive_timeout` error only once the server had finished, and the
+ *    server's own record showed the merge completing with code 0. So the job
+ *    reported completed merges as failed.
+ *
+ * So each run reads `system.parts` once per table and gives every partition
+ * holding more than one active part, or rows a lightweight delete has masked,
+ * one of the verdicts in `OPTIMISE_PARTITION_ACTION`:
+ *
+ * - `merge`: every part is on the local disk and the parts share one
+ *   projection set. The job settles it with
+ *   `OPTIMIZE TABLE … PARTITION ID '<id>' FINAL`.
+ * - `purge`: the partition carries rows a lightweight delete has masked. The
+ *   job rewrites it wherever it sits, because the rewrite is what removes
+ *   those rows from disk. This is the one case that writes to cold storage,
+ *   and it arises only when a delete has run. A delete marks its part through
+ *   `has_lightweight_delete` under `lightweight_delete_mode = 'alter_update'`,
+ *   the 26.3 default and the mode production runs (measured 2026-09-30). A
+ *   delete written as patch parts would not set it.
+ * - `skip_cold`: at least one part has moved off the local disk. The engine's
+ *   own merges settle it.
+ * - `skip_projections`: the parts carry different projection sets, which the
+ *   engine refuses to merge together (Code 388). The job says so rather than
+ *   issuing a statement that could only report a no-op as success.
+ * - `skip_partition_id`: the partition id is not purely `[0-9A-Za-z_]`, so
+ *   the job will not place it inside a statement.
+ *
+ * A partition the job skips is logged every run as a
+ * `fjall:maintenance:skipped` line. It is never a status marker: it neither
+ * pages nor defers, and the job still ends `status=ok`.
+ *
+ * A listed ReplacingMergeTree table that does not exist fails the job with
+ * `cause=missing_table`, naming the table. The enumeration alone would read a
+ * missing table as one with nothing to settle, and a dropped table left in
+ * the constants must stay as visible as it was when the whole-table statement
+ * raised `UNKNOWN_TABLE`. A missing materialised view still fails through its
+ * own statement, as `reason=optimise_<view>`.
+ *
+ * `sh` only (the image's `/bin/sh` is dash), and nothing relies on `set -e`:
+ * the TLS preamble sets `set -eu` only when TLS is active, so every failure
+ * path is an explicit `||` or `if` branch that behaves the same either way.
+ * POSIX `sh` has no local variables, and the statement runners assign `_rc`,
+ * `_label` and `_sql`, so the sweep's own variables carry a `FJALL_SWEEP_`
+ * prefix that the runners never touch.
+ */
+/** The shell function that settles one table; defined once, called per table. */
+export declare const OPTIMISE_SWEEP_FN = "fjall_ch_sweep";
+/** Log token for a partition the sweep settles. Informational, not a status. */
+export declare const OPTIMISE_PARTITION_LOG_TOKEN = "fjall:maintenance:optimise";
+/** Log token for a partition the sweep leaves to the engine. Informational,
+ *  not a status: no metric filter reads it. */
+export declare const OPTIMISE_SKIPPED_LOG_TOKEN = "fjall:maintenance:skipped";
+/** The verdicts the enumeration assigns. `merge` and `purge` issue a
+ *  statement; the rest are logged. `skip_partition_id` is the shell's own
+ *  refusal of an id it will not place inside a statement. */
+export declare const OPTIMISE_PARTITION_ACTION: {
+    readonly merge: "merge";
+    readonly purge: "purge";
+    readonly skipCold: "skip_cold";
+    readonly skipProjections: "skip_projections";
+    readonly skipPartitionId: "skip_partition_id";
+};
+export interface ClickHouseOptimiseScriptParams {
+    /** Database the listed tables live in. */
+    readonly databaseName: string;
+    /** ReplacingMergeTree tables, settled partition by partition. */
+    readonly replacingTables: readonly string[];
+    /** Materialised views and their target tables, compacted without FINAL. */
+    readonly mvTables: readonly string[];
+}
+/**
+ * The optimise job body, to follow `buildMaintenancePreamble`. Globbing is
+ * switched off because the sweep word-splits the enumeration's output; every
+ * word is already `[0-9A-Za-z_:]`, so this is defence in depth. Exits 0 with
+ * `status=ok` only when every table and view settled without a failure.
+ */
+export declare function buildClickHouseOptimiseScript(params: ClickHouseOptimiseScriptParams): string;

package/dist/lib/resources/aws/database/clickhouseOptimiseScript.js

@@ -0,0 +1,160 @@
+/**
+ * The optimise task's shell body: settle, partition by partition, the
+ * ReplacingMergeTree partitions that have something to settle, then compact
+ * the materialised views.
+ *
+ * A whole-table `OPTIMIZE TABLE t FINAL` is the wrong unit of work. These four
+ * reasons were measured on clickhouse-server 26.3.17.56, the image
+ * `CLICKHOUSE_IMAGE` pinned on 2026-09-30:
+ *
+ * 1. **It rewrites what is already settled.** FINAL merges every partition,
+ *    a partition that is already a single part included, so the work grows
+ *    with the table rather than with what changed.
+ * 2. **It rewrites cold data.** A merged part lands on the coldest volume its
+ *    sources sit on, so every cold partition is read back from object
+ *    storage and written to it again on every run.
+ * 3. **It stops silently.** At the first partition it cannot select it
+ *    returns success without touching the rest of the table.
+ * 4. **Its duration is the table's, not the change's.** The statement holds
+ *    the maintenance identity's single query slot for as long as the whole
+ *    table takes. On production, `clickhouse-client` printed its
+ *    `receive_timeout` error only once the server had finished, and the
+ *    server's own record showed the merge completing with code 0. So the job
+ *    reported completed merges as failed.
+ *
+ * So each run reads `system.parts` once per table and gives every partition
+ * holding more than one active part, or rows a lightweight delete has masked,
+ * one of the verdicts in `OPTIMISE_PARTITION_ACTION`:
+ *
+ * - `merge`: every part is on the local disk and the parts share one
+ *   projection set. The job settles it with
+ *   `OPTIMIZE TABLE … PARTITION ID '<id>' FINAL`.
+ * - `purge`: the partition carries rows a lightweight delete has masked. The
+ *   job rewrites it wherever it sits, because the rewrite is what removes
+ *   those rows from disk. This is the one case that writes to cold storage,
+ *   and it arises only when a delete has run. A delete marks its part through
+ *   `has_lightweight_delete` under `lightweight_delete_mode = 'alter_update'`,
+ *   the 26.3 default and the mode production runs (measured 2026-09-30). A
+ *   delete written as patch parts would not set it.
+ * - `skip_cold`: at least one part has moved off the local disk. The engine's
+ *   own merges settle it.
+ * - `skip_projections`: the parts carry different projection sets, which the
+ *   engine refuses to merge together (Code 388). The job says so rather than
+ *   issuing a statement that could only report a no-op as success.
+ * - `skip_partition_id`: the partition id is not purely `[0-9A-Za-z_]`, so
+ *   the job will not place it inside a statement.
+ *
+ * A partition the job skips is logged every run as a
+ * `fjall:maintenance:skipped` line. It is never a status marker: it neither
+ * pages nor defers, and the job still ends `status=ok`.
+ *
+ * A listed ReplacingMergeTree table that does not exist fails the job with
+ * `cause=missing_table`, naming the table. The enumeration alone would read a
+ * missing table as one with nothing to settle, and a dropped table left in
+ * the constants must stay as visible as it was when the whole-table statement
+ * raised `UNKNOWN_TABLE`. A missing materialised view still fails through its
+ * own statement, as `reason=optimise_<view>`.
+ *
+ * `sh` only (the image's `/bin/sh` is dash), and nothing relies on `set -e`:
+ * the TLS preamble sets `set -eu` only when TLS is active, so every failure
+ * path is an explicit `||` or `if` branch that behaves the same either way.
+ * POSIX `sh` has no local variables, and the statement runners assign `_rc`,
+ * `_label` and `_sql`, so the sweep's own variables carry a `FJALL_SWEEP_`
+ * prefix that the runners never touch.
+ */
+import { CLICKHOUSE_DEFAULT_DISK_NAME, MAINTENANCE_STATUS, maintenanceStatusMarker } from "./clickhouseConstants.js";
+import { MAINTENANCE_RUN_FN, maintenanceOkMarker, maintenanceRun } from "./clickhouseMaintenanceScript.js";
+const JOB = "optimise";
+/** The shell function that settles one table; defined once, called per table. */
+export const OPTIMISE_SWEEP_FN = "fjall_ch_sweep";
+/** Log token for a partition the sweep settles. Informational, not a status. */
+export const OPTIMISE_PARTITION_LOG_TOKEN = "fjall:maintenance:optimise";
+/** Log token for a partition the sweep leaves to the engine. Informational,
+ *  not a status: no metric filter reads it. */
+export const OPTIMISE_SKIPPED_LOG_TOKEN = "fjall:maintenance:skipped";
+/** The verdicts the enumeration assigns. `merge` and `purge` issue a
+ *  statement; the rest are logged. `skip_partition_id` is the shell's own
+ *  refusal of an id it will not place inside a statement. */
+export const OPTIMISE_PARTITION_ACTION = {
+    merge: "merge",
+    purge: "purge",
+    skipCold: "skip_cold",
+    skipProjections: "skip_projections",
+    skipPartitionId: "skip_partition_id"
+};
+const IDENTIFIER_PATTERN = /^[a-z][a-z0-9_]*$/;
+function assertIdentifier(kind, value) {
+    if (!IDENTIFIER_PATTERN.test(value)) {
+        throw new Error(`ClickHouse optimise ${kind} must match ${IDENTIFIER_PATTERN}.`);
+    }
+}
+/**
+ * One row per partition that needs attention, as `<id>:<verdict>`.
+ *
+ * A partition id that is not purely `[0-9A-Za-z_]` is never returned as
+ * itself: it comes back hex-encoded with the `skip_partition_id` verdict, so
+ * nothing but an id the statement can quote safely ever reaches the shell.
+ * Every partition key in the webapp schema yields a digits-only id, so this
+ * arm exists for a future key rather than a present one.
+ */
+function enumerationQuery(databaseName, table) {
+    const safeId = "length(partition_id) > 0 AND NOT match(partition_id, '[^0-9A-Za-z_]')";
+    const verdict = `multiIf(NOT (${safeId}), '${OPTIMISE_PARTITION_ACTION.skipPartitionId}',` +
+        ` uniqExact(arrayStringConcat(arraySort(projections), ',')) > 1, '${OPTIMISE_PARTITION_ACTION.skipProjections}',` +
+        ` max(has_lightweight_delete) = 1, '${OPTIMISE_PARTITION_ACTION.purge}',` +
+        ` countIf(disk_name != '${CLICKHOUSE_DEFAULT_DISK_NAME}') > 0, '${OPTIMISE_PARTITION_ACTION.skipCold}',` +
+        ` '${OPTIMISE_PARTITION_ACTION.merge}')`;
+    return (`SELECT concat(if(${safeId}, partition_id, concat('hex_', hex(partition_id))), ':', ${verdict})` +
+        ` FROM system.parts WHERE database = '${databaseName}' AND table = '${table}' AND active` +
+        ` GROUP BY partition_id HAVING count() > 1 OR max(has_lightweight_delete) = 1` +
+        ` ORDER BY partition_id DESC FORMAT TSVRaw`);
+}
+function sweepFunction(databaseName) {
+    const failedMissing = `echo "${maintenanceStatusMarker(MAINTENANCE_STATUS.failed)} reason=optimise_$FJALL_SWEEP_TABLE cause=missing_table job=$FJALL_JOB" >&2`;
+    const existsQuery = `SELECT count() FROM system.tables WHERE database = '${databaseName}' AND name = '$FJALL_SWEEP_TABLE'`;
+    const enumeration = enumerationQuery(databaseName, "$FJALL_SWEEP_TABLE");
+    const statement = `OPTIMIZE TABLE ${databaseName}.$FJALL_SWEEP_TABLE PARTITION ID '$FJALL_SWEEP_ID' FINAL`;
+    const { merge, purge, skipPartitionId } = OPTIMISE_PARTITION_ACTION;
+    return [
+        `${OPTIMISE_SWEEP_FN}() {`,
+        `  FJALL_SWEEP_TABLE="$1"; FJALL_SWEEP_RC=0`,
+        `  FJALL_SWEEP_EXISTS=$(${MAINTENANCE_RUN_FN} "exists_$FJALL_SWEEP_TABLE" "${existsQuery}") || return 1`,
+        `  if [ "$FJALL_SWEEP_EXISTS" != 1 ]; then ${failedMissing}; return 1; fi`,
+        `  FJALL_SWEEP_PARTS=$(${MAINTENANCE_RUN_FN} "enumerate_$FJALL_SWEEP_TABLE" "${enumeration}") || return 1`,
+        `  for FJALL_SWEEP_ENTRY in $FJALL_SWEEP_PARTS; do`,
+        `    FJALL_SWEEP_ID=\${FJALL_SWEEP_ENTRY%%:*}; FJALL_SWEEP_ACTION=\${FJALL_SWEEP_ENTRY#*:}`,
+        `    case "$FJALL_SWEEP_ID" in ''|*[!0-9A-Za-z_]*) FJALL_SWEEP_ID=unreadable; FJALL_SWEEP_ACTION=${skipPartitionId} ;; esac`,
+        `    case "$FJALL_SWEEP_ACTION" in`,
+        `      ${merge}|${purge})`,
+        `        echo "${OPTIMISE_PARTITION_LOG_TOKEN} table=$FJALL_SWEEP_TABLE partition=$FJALL_SWEEP_ID action=$FJALL_SWEEP_ACTION job=$FJALL_JOB" >&2`,
+        `        ${MAINTENANCE_RUN_FN} "optimise_$FJALL_SWEEP_TABLE" "${statement}" || FJALL_SWEEP_RC=1 ;;`,
+        `      *)`,
+        `        echo "${OPTIMISE_SKIPPED_LOG_TOKEN} table=$FJALL_SWEEP_TABLE partition=$FJALL_SWEEP_ID reason=$FJALL_SWEEP_ACTION job=$FJALL_JOB" >&2 ;;`,
+        `    esac`,
+        `  done`,
+        `  return "$FJALL_SWEEP_RC"`,
+        `}`
+    ].join("\n");
+}
+/**
+ * The optimise job body, to follow `buildMaintenancePreamble`. Globbing is
+ * switched off because the sweep word-splits the enumeration's output; every
+ * word is already `[0-9A-Za-z_:]`, so this is defence in depth. Exits 0 with
+ * `status=ok` only when every table and view settled without a failure.
+ */
+export function buildClickHouseOptimiseScript(params) {
+    const { databaseName, replacingTables, mvTables } = params;
+    assertIdentifier("database name", databaseName);
+    for (const table of [...replacingTables, ...mvTables]) {
+        assertIdentifier("table name", table);
+    }
+    return [
+        "set -f",
+        "FAILED=0",
+        sweepFunction(databaseName),
+        ...replacingTables.map((table) => `${OPTIMISE_SWEEP_FN} ${table} || FAILED=1`),
+        ...mvTables.map((table) => `${maintenanceRun(`optimise_${table}`, `OPTIMIZE TABLE ${databaseName}.${table}`)} || FAILED=1`),
+        `if [ "$FAILED" = 0 ]; then ${maintenanceOkMarker(JOB)}; fi`,
+        'exit "$FAILED"'
+    ].join("\n");
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fjall/components-infrastructure",
-  "version": "39.0.3",
+  "version": "39.0.4",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/fjall-tech/fjall.git",
@@ -80,8 +80,8 @@
   },
   "dependencies": {
     "@aws-sdk/client-organizations": "^3.1098.0",
-    "@fjall/generator": "^39.0.3",
-    "@fjall/util": "^39.0.3",
+    "@fjall/generator": "^39.0.4",
+    "@fjall/util": "^39.0.4",
     "constructs": "^10.7.2",
     "zod": "^4.4.3"
   },