@fjall/components-infrastructure 39.0.1 → 39.0.2

package/dist/lib/resources/aws/compute/persistentDataVolumeLambda.source.cjs

@@ -26,16 +26,25 @@
  *      Lambda-Errors alarm (a tag-uniqueness invariant violation; silently
  *      picking one risks attaching the wrong data and requires operator
  *      investigation).
- *   3. AttachVolume(Device: DEVICE_NAME, InstanceId, VolumeId).
- *   4. Poll DescribeVolumes until State === "in-use" AND the matching
+ *   3. Wait for the volume to be released: poll DescribeVolumes until its
+ *      State is "available", deadline RELEASE_DEADLINE_MS. Timeout →
+ *      ABANDON; the ASG launches another instance and the wait starts over.
+ *   4. AttachVolume(Device: DEVICE_NAME, InstanceId, VolumeId). If EC2
+ *      refuses because a concurrent delivery of the same event is already
+ *      attaching the volume here, carry on to the attach wait.
+ *   5. Poll DescribeVolumes until State === "in-use" AND the matching
  *      Attachments[].State === "attached", deadline 90s. Timeout → ABANDON.
- *   5. CompleteLifecycleAction(CONTINUE) on success.
+ *   6. CompleteLifecycleAction(CONTINUE) on success. A token whose action
+ *      has already ended is logged as
+ *      `event: "lifecycle_action_already_ended"`, not retried.
  *
- * The Lambda is paired with the TERMINATING-side DetachVolume branch in
- * ec2GracefulTerminationLambda.source.cjs. The two Lambdas form a closed
- * re-attach contract: TERMINATING detaches the tagged volume from the
- * outgoing instance, LAUNCHING (this file) re-attaches it to the incoming
- * instance.
+ * Nothing detaches the volume. The instance that holds it releases it when
+ * it terminates, after the TERMINATING handler
+ * (ec2GracefulTerminationLambda.source.cjs) has drained its tasks, so a
+ * detach can never pull the filesystem out from under a running database.
+ * This handler therefore waits for the release instead of forcing it.
+ * RELEASE_DEADLINE_MS covers the TERMINATING handler's drain and StopTask
+ * budgets plus the instance's shutdown; a test pins it to them.
  *
  * CommonJS rather than ESM because Code.fromInline lands the source as
  * `index.js`, which Lambda treats as CommonJS by default.
@@ -51,18 +60,41 @@ const {
 } = require("@aws-sdk/client-auto-scaling");
 
 const ATTACH_POLL_INTERVAL_MS = 5_000;
-// 90s attach deadline. AWS volume attach is usually < 30s; the bootstrap
-// script waits 60s for /dev/xvdf so the hook MUST complete inside that.
+// AWS volume attach is usually < 30s.
 const ATTACH_DEADLINE_MS = 90_000;
+const RELEASE_POLL_INTERVAL_MS = 10_000;
+// The TERMINATING handler's drain (240 s) and StopTask (150 s) budgets, plus
+// its network cleanup and the instance's OS shutdown.
+const RELEASE_DEADLINE_MS = 480_000;
 const TAG_OWNER_LOGICAL_ID = "fjall:OwnerLogicalId";
 const TAG_LIFECYCLE = "fjall:Lifecycle";
 const TAG_LIFECYCLE_VALUE = "data-volume";
 const TAG_STACK_ID = "fjall:StackId";
+// Bounds every AWS call, SDK retries included, so a hung request cannot carry
+// an attempt past the Lambda's timeout.
+const SDK_CALL_TIMEOUT_MS = 20_000;
+const LIFECYCLE_EVENT = {
+  ALREADY_ENDED: "lifecycle_action_already_ended"
+};
+const LIFECYCLE_OUTCOME = {
+  COMPLETED: "completed",
+  ALREADY_ENDED: "already_ended"
+};
 
 function errMessage(err) {
   return err && err.message ? err.message : String(err);
 }
 
+function send(client, command) {
+  return client.send(command, {
+    abortSignal: AbortSignal.timeout(SDK_CALL_TIMEOUT_MS)
+  });
+}
+
+function consoleLog(msg, fields) {
+  console.log(JSON.stringify({ message: msg, ...(fields || {}) }));
+}
+
 let _ec2;
 let _asg;
 function getEc2() {
@@ -106,7 +138,8 @@ function readEnv(env, name) {
 }
 
 async function findOwnedVolume(ec2Client, ownerLogicalId, stackId) {
-  const result = await ec2Client.send(
+  const result = await send(
+    ec2Client,
     new DescribeVolumesCommand({
       Filters: [
         { Name: `tag:${TAG_OWNER_LOGICAL_ID}`, Values: [ownerLogicalId] },
@@ -119,6 +152,57 @@ async function findOwnedVolume(ec2Client, ownerLogicalId, stackId) {
   return volumes;
 }
 
+/**
+ * Poll until the volume can be attached to `instanceId`. Resolves
+ * `{ released: true, attachedHere: false }` once it is "available",
+ * `{ released: true, attachedHere: true }` if an earlier delivery of this
+ * lifecycle action already attached it here, and `{ released: false, ... }`
+ * at the deadline. A failed read is retried until the deadline and named in
+ * the result, so a timeout caused by an unreadable volume says so.
+ */
+async function pollUntilReleased(
+  ec2Client,
+  volumeId,
+  instanceId,
+  sleepFn,
+  nowFn
+) {
+  const deadline = nowFn() + RELEASE_DEADLINE_MS;
+  let lastState;
+  let lastError;
+  for (;;) {
+    try {
+      const desc = await send(
+        ec2Client,
+        new DescribeVolumesCommand({ VolumeIds: [volumeId] })
+      );
+      const volume =
+        desc.Volumes && desc.Volumes[0] ? desc.Volumes[0] : undefined;
+      lastError = undefined;
+      lastState = volume ? volume.State : undefined;
+      if (volume && volume.State === "available") {
+        return { released: true, attachedHere: false };
+      }
+      const attachedHere = (volume?.Attachments || []).some(
+        (a) => a.InstanceId === instanceId && a.State === "attached"
+      );
+      if (attachedHere) {
+        return { released: true, attachedHere: true };
+      }
+    } catch (err) {
+      lastError = errMessage(err);
+    }
+    if (nowFn() >= deadline) {
+      return {
+        released: false,
+        lastState,
+        ...(lastError !== undefined && { error: lastError })
+      };
+    }
+    await sleepFn(Math.min(RELEASE_POLL_INTERVAL_MS, deadline - nowFn()));
+  }
+}
+
 async function pollUntilAttached(
   ec2Client,
   volumeId,
@@ -127,8 +211,9 @@ async function pollUntilAttached(
   nowFn
 ) {
   const deadline = nowFn() + ATTACH_DEADLINE_MS;
-  while (nowFn() < deadline) {
-    const desc = await ec2Client.send(
+  for (;;) {
+    const desc = await send(
+      ec2Client,
       new DescribeVolumesCommand({ VolumeIds: [volumeId] })
     );
     const volume =
@@ -141,9 +226,45 @@ async function pollUntilAttached(
         return true;
       }
     }
-    await sleepFn(ATTACH_POLL_INTERVAL_MS);
+    if (nowFn() >= deadline) return false;
+    await sleepFn(Math.min(ATTACH_POLL_INTERVAL_MS, deadline - nowFn()));
+  }
+}
+
+// Whether a refused AttachVolume lost a race with a concurrent delivery of
+// the same event, which is attaching the volume to this instance.
+async function attachingHere(ec2Client, volumeId, instanceId, log) {
+  try {
+    const desc = await send(
+      ec2Client,
+      new DescribeVolumesCommand({ VolumeIds: [volumeId] })
+    );
+    const volume =
+      desc.Volumes && desc.Volumes[0] ? desc.Volumes[0] : undefined;
+    return (volume?.Attachments || []).some(
+      (a) =>
+        a.InstanceId === instanceId &&
+        (a.State === "attaching" || a.State === "attached")
+    );
+  } catch (err) {
+    log("Volume read after a refused attach failed", {
+      volumeId,
+      instanceId,
+      error: errMessage(err)
+    });
+    return false;
   }
-  return false;
+}
+
+// Auto Scaling refuses a token whose action has already ended, because
+// another delivery of the event completed it or the heartbeat expired, with
+// a ValidationError naming no active lifecycle action.
+function isNoActiveLifecycleAction(err) {
+  return (
+    Boolean(err) &&
+    err.name === "ValidationError" &&
+    /No active Lifecycle Action/i.test(errMessage(err))
+  );
 }
 
 async function completeLifecycle(
@@ -153,13 +274,28 @@ async function completeLifecycle(
   actionToken,
   result
 ) {
-  await asgClient.send(
-    new CompleteLifecycleActionCommand({
-      AutoScalingGroupName: asgName,
-      LifecycleHookName: hookName,
-      LifecycleActionToken: actionToken,
-      LifecycleActionResult: result
-    })
+  try {
+    await send(
+      asgClient,
+      new CompleteLifecycleActionCommand({
+        AutoScalingGroupName: asgName,
+        LifecycleHookName: hookName,
+        LifecycleActionToken: actionToken,
+        LifecycleActionResult: result
+      })
+    );
+    return LIFECYCLE_OUTCOME.COMPLETED;
+  } catch (err) {
+    if (isNoActiveLifecycleAction(err)) return LIFECYCLE_OUTCOME.ALREADY_ENDED;
+    throw err;
+  }
+}
+
+function logLifecycleOutcome(log, outcome, instanceId) {
+  if (outcome !== LIFECYCLE_OUTCOME.ALREADY_ENDED) return;
+  log(
+    "The lifecycle action had already ended: another delivery completed it, or its heartbeat expired",
+    { event: LIFECYCLE_EVENT.ALREADY_ENDED, instanceId }
   );
 }
 
@@ -171,11 +307,7 @@ async function processRecord(record, deps) {
     ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
   const nowFn = (deps && deps.now) || (() => Date.now());
   const env = (deps && deps.env) || process.env;
-  const log =
-    (deps && deps.log) ||
-    ((msg, fields) => {
-      console.log(JSON.stringify({ message: msg, ...(fields || {}) }));
-    });
+  const log = (deps && deps.log) || consoleLog;
 
   const parsed = parseLifecycleMessage(record);
   if (!parsed.valid) {
@@ -185,6 +317,18 @@ async function processRecord(record, deps) {
     return;
   }
   const { instanceId, asgName, actionToken, hookName } = parsed;
+  const finish = async (result) =>
+    logLifecycleOutcome(
+      log,
+      await completeLifecycle(
+        asgClient,
+        asgName,
+        hookName,
+        actionToken,
+        result
+      ),
+      instanceId
+    );
 
   const ownerLogicalId = readEnv(env, "OWNER_LOGICAL_ID");
   const stackId = readEnv(env, "STACK_ID");
@@ -198,13 +342,7 @@ async function processRecord(record, deps) {
       asgName,
       instanceId
     });
-    await completeLifecycle(
-      asgClient,
-      asgName,
-      hookName,
-      actionToken,
-      "ABANDON"
-    );
+    await finish("ABANDON");
     return;
   }
 
@@ -217,13 +355,7 @@ async function processRecord(record, deps) {
       instanceId,
       error: errMessage(err)
     });
-    await completeLifecycle(
-      asgClient,
-      asgName,
-      hookName,
-      actionToken,
-      "ABANDON"
-    );
+    await finish("ABANDON");
     return;
   }
 
@@ -232,13 +364,7 @@ async function processRecord(record, deps) {
       ownerLogicalId,
       instanceId
     });
-    await completeLifecycle(
-      asgClient,
-      asgName,
-      hookName,
-      actionToken,
-      "ABANDON"
-    );
+    await finish("ABANDON");
     return;
   }
 
@@ -250,41 +376,44 @@ async function processRecord(record, deps) {
         volumeIds: volumes.map((v) => v.VolumeId)
       }
     );
-    await completeLifecycle(
-      asgClient,
-      asgName,
-      hookName,
-      actionToken,
-      "ABANDON"
-    );
+    await finish("ABANDON");
     return;
   }
 
-  const volume = volumes[0];
-  const volumeId = volume.VolumeId;
+  const volumeId = volumes[0].VolumeId;
 
-  // Already attached to the launching instance (e.g. previous Lambda invocation
-  // succeeded before the hook timeout fired and ASG re-fired the message).
-  const existingAttachment = (volume.Attachments || []).find(
-    (a) => a.InstanceId === instanceId && a.State === "attached"
+  // An earlier delivery of this lifecycle action may already have attached
+  // the volume here; the release wait reports that as `attachedHere`.
+  const release = await pollUntilReleased(
+    ec2Client,
+    volumeId,
+    instanceId,
+    sleepFn,
+    nowFn
   );
-  if (existingAttachment !== undefined) {
+  if (!release.released) {
+    log("Volume was not released within the deadline; abandoning launch", {
+      volumeId,
+      instanceId,
+      deadlineMs: RELEASE_DEADLINE_MS,
+      lastState: release.lastState,
+      ...(release.error !== undefined && { error: release.error })
+    });
+    await finish("ABANDON");
+    return;
+  }
+  if (release.attachedHere) {
     log("Volume already attached to launching instance; completing CONTINUE", {
       volumeId,
       instanceId
     });
-    await completeLifecycle(
-      asgClient,
-      asgName,
-      hookName,
-      actionToken,
-      "CONTINUE"
-    );
+    await finish("CONTINUE");
     return;
   }
 
   try {
-    await ec2Client.send(
+    await send(
+      ec2Client,
       new AttachVolumeCommand({
         VolumeId: volumeId,
         InstanceId: instanceId,
@@ -292,19 +421,21 @@ async function processRecord(record, deps) {
       })
     );
   } catch (err) {
-    log("AttachVolume failed; abandoning", {
-      volumeId,
-      instanceId,
-      error: errMessage(err)
-    });
-    await completeLifecycle(
-      asgClient,
-      asgName,
-      hookName,
-      actionToken,
-      "ABANDON"
-    );
-    return;
+    if (await attachingHere(ec2Client, volumeId, instanceId, log)) {
+      log("AttachVolume was refused, but the volume is attaching here", {
+        volumeId,
+        instanceId,
+        error: errMessage(err)
+      });
+    } else {
+      log("AttachVolume failed; abandoning", {
+        volumeId,
+        instanceId,
+        error: errMessage(err)
+      });
+      await finish("ABANDON");
+      return;
+    }
   }
 
   const attached = await pollUntilAttached(
@@ -323,13 +454,7 @@ async function processRecord(record, deps) {
         deadlineMs: ATTACH_DEADLINE_MS
       }
     );
-    await completeLifecycle(
-      asgClient,
-      asgName,
-      hookName,
-      actionToken,
-      "ABANDON"
-    );
+    await finish("ABANDON");
     return;
   }
 
@@ -337,13 +462,7 @@ async function processRecord(record, deps) {
     volumeId,
     instanceId
   });
-  await completeLifecycle(
-    asgClient,
-    asgName,
-    hookName,
-    actionToken,
-    "CONTINUE"
-  );
+  await finish("CONTINUE");
 }
 
 exports.handler = async (event) => {
@@ -362,12 +481,16 @@ exports.handler = async (event) => {
       try {
         const parsed = parseLifecycleMessage(record);
         if (!parsed.valid) continue;
-        await completeLifecycle(
-          getAsg(),
-          parsed.asgName,
-          parsed.hookName,
-          parsed.actionToken,
-          "ABANDON"
+        logLifecycleOutcome(
+          consoleLog,
+          await completeLifecycle(
+            getAsg(),
+            parsed.asgName,
+            parsed.hookName,
+            parsed.actionToken,
+            "ABANDON"
+          ),
+          parsed.instanceId
         );
       } catch (innerErr) {
         console.error(
@@ -386,13 +509,21 @@ exports._internals = {
   parseLifecycleMessage,
   readEnv,
   findOwnedVolume,
+  pollUntilReleased,
   pollUntilAttached,
+  attachingHere,
+  isNoActiveLifecycleAction,
   completeLifecycle,
   processRecord,
   ATTACH_POLL_INTERVAL_MS,
   ATTACH_DEADLINE_MS,
+  RELEASE_POLL_INTERVAL_MS,
+  RELEASE_DEADLINE_MS,
   TAG_OWNER_LOGICAL_ID,
   TAG_LIFECYCLE,
   TAG_LIFECYCLE_VALUE,
-  TAG_STACK_ID
+  TAG_STACK_ID,
+  SDK_CALL_TIMEOUT_MS,
+  LIFECYCLE_EVENT,
+  LIFECYCLE_OUTCOME
 };

package/dist/lib/resources/aws/database/clickhouseUserData.js

@@ -1,3 +1,4 @@
+import { PERSISTENT_DATA_VOLUME_LAUNCHING_HEARTBEAT_SECONDS } from "../compute/persistentDataVolume.js";
 import { CLICKHOUSE_DATA_MOUNT_PATH, CLICKHOUSE_EBS_DEVICE_NAME, CLICKHOUSE_CONFIG_SUBDIR, CLICKHOUSE_USERS_SUBDIR, CLICKHOUSE_ECS_RESERVED_MEMORY_MIB, CLICKHOUSE_HOST_METRICS, CLICKHOUSE_HTTP_PORT, CLICKHOUSE_HTTPS_PORT, CLICKHOUSE_TCP_SECURE_PORT, CLICKHOUSE_TLS_CERT_MOUNT_PATH, CLICKHOUSE_PROMETHEUS_PORT, CLICKHOUSE_UID, clickHousePasswordSha256Snippet } from "./clickhouseConstants.js";
 import { deriveClickHouseServerTuning, MAX_SERVER_MEMORY_RAM_RATIO, SYSTEM_LOG_TTL } from "./clickhouseTuning.js";
 import { renderUsersXml } from "./clickhouseXmlRenderer.js";
@@ -462,7 +463,7 @@ mkdir -p /etc/systemd/system/docker.service.d
 cat > /etc/systemd/system/docker.service.d/override.conf << 'DOCKEROVERRIDE'
 [Service]
 LimitNOFILE=1048576
-LimitNPROC=unlimited
+LimitNPROC=infinity
 DOCKEROVERRIDE
 systemctl daemon-reload
 systemctl restart docker || true
@@ -475,8 +476,10 @@ systemctl restart docker || true
 mkdir -p /etc/ecs
 echo "ECS_RESERVED_MEMORY=${CLICKHOUSE_ECS_RESERVED_MEMORY_MIB}" >> /etc/ecs/ecs.config
 
-# Wait for the EBS volume to be attached (up to 60 seconds)
-for i in $(seq 1 60); do
+# Wait for the EBS volume. The launching lifecycle hook attaches it only once
+# the previous instance has released it, and ABANDONs this instance when its
+# heartbeat expires, so the wait matches the heartbeat.
+for _ in $(seq 1 ${PERSISTENT_DATA_VOLUME_LAUNCHING_HEARTBEAT_SECONDS}); do
   if [ -b "$DEVICE" ]; then
     break
   fi
@@ -484,13 +487,53 @@ for i in $(seq 1 60); do
 done
 
 if [ ! -b "$DEVICE" ]; then
-  echo "ERROR: EBS volume $DEVICE not found after 60 seconds" >&2
+  echo "ERROR: EBS volume $DEVICE not found after ${PERSISTENT_DATA_VOLUME_LAUNCHING_HEARTBEAT_SECONDS} seconds" >&2
   exit 1
 fi
 
-# Format only if not already formatted (idempotent)
-if ! blkid "$DEVICE" | grep -q ext4; then
+# Format only a device that holds no filesystem this construct could have
+# made. blkid -p exits 2 both for a blank device and for one it could not
+# read, so the device must also read back, and carry the ext4 magic neither
+# in the primary superblock nor in the first backup (group 1 of a 4 KiB-block
+# filesystem, the one e2fsck -b 32768 reads). Content is not a test: a new EBS
+# volume need not read back as zeros. Random bytes carry the magic at a given
+# offset once in 65,536 reads, and such a new volume is refused too, which is
+# the side to err on. Anything else is refused: formatting is the one step
+# here that can destroy the database.
+# Prints, as hex, the two bytes at offset $2 of the 4 KiB block $1; fails
+# unless the whole block was read and parsed.
+read_magic() {
+  local block
+  block=$(mktemp) || return 1
+  if ! dd if="$DEVICE" of="$block" bs=4096 skip="$1" count=1 iflag=direct status=none || [ "$(stat -c %s "$block")" != 4096 ]; then
+    rm -f "$block"
+    return 1
+  fi
+  od -An -tx1 -j "$2" -N 2 "$block" | tr -d ' \\n'
+  local rc=$?
+  rm -f "$block"
+  return "$rc"
+}
+BLKID_RC=0
+DATA_VOLUME_FS_TYPE=$(blkid -p -s TYPE -o value "$DEVICE") || BLKID_RC=$?
+if [ "$BLKID_RC" -eq 0 ]; then
+  if [ "$DATA_VOLUME_FS_TYPE" != "ext4" ]; then
+    echo "ERROR: $DEVICE holds '$DATA_VOLUME_FS_TYPE', not an ext4 filesystem; refusing to mount or format it" >&2
+    exit 1
+  fi
+elif [ "$BLKID_RC" -eq 2 ]; then
+  if ! PRIMARY_MAGIC=$(read_magic 0 1080) || ! BACKUP_MAGIC=$(read_magic 32768 56); then
+    echo "ERROR: blkid found no filesystem on $DEVICE, but its superblocks could not be read; refusing to format it" >&2
+    exit 1
+  fi
+  if [ "$PRIMARY_MAGIC" = 53ef ] || [ "$BACKUP_MAGIC" = 53ef ]; then
+    echo "ERROR: blkid found no filesystem on $DEVICE, but it carries an ext4 superblock (primary '$PRIMARY_MAGIC', backup '$BACKUP_MAGIC'); refusing to format it (e2fsck -b 32768 $DEVICE repairs one whose primary superblock is damaged)" >&2
+    exit 1
+  fi
   mkfs.ext4 -m 0 "$DEVICE"
+else
+  echo "ERROR: blkid could not probe $DEVICE (exit $BLKID_RC); refusing to mount or format it" >&2
+  exit 1
 fi
 
 # Mount

package/dist/lib/resources/aws/monitoring/metricNamespaces.d.ts

@@ -4,10 +4,13 @@
  * Log-pattern metric filters and their alarms read from these namespaces. The
  * webapp app service declares RLS alarms under `WEBAPP` and relocates the
  * ClickHouse stuck-merge alarm under `CLICKHOUSE` (the log line is app-emitted,
- * so it is an app-log alarm carrying a database-domain namespace).
+ * so it is an app-log alarm carrying a database-domain namespace). The EC2
+ * graceful-termination handler alarms under `EC2` when it cannot prove a
+ * terminating instance's ECS tasks stopped.
  */
 export declare const METRIC_NAMESPACE: {
     readonly CLICKHOUSE: "Fjall/ClickHouse";
+    readonly EC2: "Fjall/EC2";
     readonly ECS: "Fjall/ECS";
     readonly WEBAPP: "Fjall/WebApp";
 };

package/dist/lib/resources/aws/monitoring/metricNamespaces.js

@@ -5,10 +5,13 @@ import { Aws } from "aws-cdk-lib";
  * Log-pattern metric filters and their alarms read from these namespaces. The
  * webapp app service declares RLS alarms under `WEBAPP` and relocates the
  * ClickHouse stuck-merge alarm under `CLICKHOUSE` (the log line is app-emitted,
- * so it is an app-log alarm carrying a database-domain namespace).
+ * so it is an app-log alarm carrying a database-domain namespace). The EC2
+ * graceful-termination handler alarms under `EC2` when it cannot prove a
+ * terminating instance's ECS tasks stopped.
  */
 export const METRIC_NAMESPACE = {
     CLICKHOUSE: "Fjall/ClickHouse",
+    EC2: "Fjall/EC2",
     ECS: "Fjall/ECS",
     WEBAPP: "Fjall/WebApp"
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fjall/components-infrastructure",
-  "version": "39.0.1",
+  "version": "39.0.2",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/fjall-tech/fjall.git",
@@ -80,8 +80,8 @@
   },
   "dependencies": {
     "@aws-sdk/client-organizations": "^3.1098.0",
-    "@fjall/generator": "^39.0.1",
-    "@fjall/util": "^39.0.1",
+    "@fjall/generator": "^39.0.2",
+    "@fjall/util": "^39.0.2",
     "constructs": "^10.7.2",
     "zod": "^4.4.3"
   },