@fjall/components-infrastructure 0.95.0 → 0.99.1

package/dist/lib/resources/aws/compute/ec2GracefulTerminationHandler.d.ts

@@ -0,0 +1,41 @@
+import { type AutoScalingGroup } from "aws-cdk-lib/aws-autoscaling";
+import { Construct } from "constructs";
+import { LambdaFunction } from "./lambda.js";
+import { SQSQueue } from "../messaging/sqs.js";
+/**
+ * Description prefix baked into the Lambda + execution-role descriptions.
+ * Tests grep CFN output for this needle to locate the function across
+ * stacks that may contain unrelated Lambdas — keep it here as the single
+ * source of truth so a rename does not silently break the test query.
+ */
+export declare const EC2_GRACEFUL_TERMINATION_HANDLER_DESCRIPTION = "EC2 graceful termination handler";
+export interface Ec2GracefulTerminationHandlerProps {
+    /** ASG to attach the EC2_INSTANCE_TERMINATING hook to. */
+    autoScalingGroup: AutoScalingGroup;
+    /**
+     * ECS cluster ARN — when set, the Lambda drains and deregisters the
+     * container instance before generic cleanup. Empty string is normalised
+     * to `undefined`; consumers should pass `undefined` for bare-EC2
+     * deployments (bastion, Fivetran).
+     */
+    ecsClusterArn?: string;
+    /**
+     * `fjall:OwnerLogicalId` of a paired `PersistentDataVolume`. When set, the
+     * Lambda detaches the tagged volume from the terminating instance before
+     * `CompleteLifecycleAction`. Empty string is normalised to `undefined`.
+     * Pass the `PersistentDataVolume.ownerLogicalId` so the TERMINATING and
+     * LAUNCHING handlers locate the same volume.
+     */
+    dataVolumeOwnerLogicalId?: string;
+}
+/**
+ * ASG `EC2_INSTANCE_TERMINATING` hook + Lambda that drains the instance
+ * cleanly. Drain ownership lives at the EC2 layer; ECS-specific drain plugs
+ * in via `ecsClusterArn` so a single hook handles both bare-EC2 (bastion,
+ * Fivetran) and ECS-wired ASGs (ClickHouse).
+ */
+export declare class Ec2GracefulTerminationHandler extends Construct {
+    readonly lambda: LambdaFunction;
+    readonly queue: SQSQueue;
+    constructor(scope: Construct, id: string, props: Ec2GracefulTerminationHandlerProps);
+}

package/dist/lib/resources/aws/compute/ec2GracefulTerminationHandler.js

@@ -0,0 +1,194 @@
+import { readFileSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { Aws, Stack } from "aws-cdk-lib";
+import { Code, Runtime } from "aws-cdk-lib/aws-lambda";
+import { PolicyStatement, Effect } from "aws-cdk-lib/aws-iam";
+import { RetentionDays } from "aws-cdk-lib/aws-logs";
+import { LifecycleTransition, DefaultResult } from "aws-cdk-lib/aws-autoscaling";
+import { Construct } from "constructs";
+import { attachInlineAsgLifecycleHook } from "./asgInlineLifecycleHook.js";
+import { LambdaFunction } from "./lambda.js";
+import { PERSISTENT_DATA_VOLUME_TAG_LIFECYCLE, PERSISTENT_DATA_VOLUME_TAG_LIFECYCLE_VALUE, PERSISTENT_DATA_VOLUME_TAG_STACK_ID } from "./persistentDataVolume.js";
+import { SQSQueue } from "../messaging/sqs.js";
+import { Subscription } from "../messaging/subscription.js";
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const LAMBDA_SOURCE_FILE = "ec2GracefulTerminationLambda.source.cjs";
+/**
+ * Description prefix baked into the Lambda + execution-role descriptions.
+ * Tests grep CFN output for this needle to locate the function across
+ * stacks that may contain unrelated Lambdas — keep it here as the single
+ * source of truth so a rename does not silently break the test query.
+ */
+export const EC2_GRACEFUL_TERMINATION_HANDLER_DESCRIPTION = "EC2 graceful termination handler";
+const LAMBDA_TIMEOUT_SECONDS = 300;
+// Heartbeat must comfortably exceed the Lambda's drain (240s) + detach (60s)
+// budget so the hook does not time out mid-cleanup when PDV is wired.
+const HOOK_HEARTBEAT_SECONDS = 600;
+const QUEUE_VISIBILITY_TIMEOUT_SECONDS = 360;
+/**
+ * ASG `EC2_INSTANCE_TERMINATING` hook + Lambda that drains the instance
+ * cleanly. Drain ownership lives at the EC2 layer; ECS-specific drain plugs
+ * in via `ecsClusterArn` so a single hook handles both bare-EC2 (bastion,
+ * Fivetran) and ECS-wired ASGs (ClickHouse).
+ */
+export class Ec2GracefulTerminationHandler extends Construct {
+    lambda;
+    queue;
+    constructor(scope, id, props) {
+        super(scope, id);
+        const sourcePath = path.resolve(__dirname, LAMBDA_SOURCE_FILE);
+        const source = readFileSync(sourcePath, "utf-8");
+        const ecsClusterArn = resolveOptionalString(props.ecsClusterArn);
+        const dataVolumeOwnerLogicalId = resolveOptionalString(props.dataVolumeOwnerLogicalId);
+        this.queue = new SQSQueue(this, `${id}Queue`, {
+            visibilityTimeout: QUEUE_VISIBILITY_TIMEOUT_SECONDS,
+            deadLetterQueue: { enabled: true, maxReceiveCount: 5 }
+        });
+        const ecsPolicies = ecsClusterArn !== undefined
+            ? [
+                new PolicyStatement({
+                    effect: Effect.ALLOW,
+                    actions: ["ecs:ListContainerInstances"],
+                    resources: [ecsClusterArn]
+                }),
+                new PolicyStatement({
+                    effect: Effect.ALLOW,
+                    actions: [
+                        "ecs:DescribeContainerInstances",
+                        "ecs:UpdateContainerInstancesState",
+                        "ecs:DeregisterContainerInstance"
+                    ],
+                    resources: ["*"],
+                    conditions: {
+                        ArnEquals: { "ecs:cluster": ecsClusterArn }
+                    }
+                })
+            ]
+            : [];
+        const ec2Policies = new PolicyStatement({
+            effect: Effect.ALLOW,
+            actions: [
+                "ec2:DescribeAddresses",
+                "ec2:DisassociateAddress",
+                "ec2:DescribeNetworkInterfaces",
+                "ec2:DetachNetworkInterface"
+            ],
+            resources: ["*"]
+        });
+        const elbReadPolicy = new PolicyStatement({
+            effect: Effect.ALLOW,
+            actions: [
+                "elasticloadbalancing:DescribeTargetGroups",
+                "elasticloadbalancing:DescribeTargetHealth"
+            ],
+            resources: ["*"]
+        });
+        const elbDeregisterPolicy = new PolicyStatement({
+            effect: Effect.ALLOW,
+            actions: ["elasticloadbalancing:DeregisterTargets"],
+            resources: ["*"],
+            conditions: {
+                StringEquals: {
+                    "aws:ResourceTag/aws:cloudformation:stack-id": Aws.STACK_ID
+                }
+            }
+        });
+        const stack = Stack.of(this);
+        // Account/region-scoped wildcard rather than the specific ASG ARN —
+        // see PersistentDataVolume for the full deadlock writeup. Same gotcha:
+        // a specific ARN creates a CFN Ref to the ASG, the Lambda's IAM Policy
+        // waits for ASG CREATE_COMPLETE, the ASG can't complete without this
+        // Lambda's CompleteLifecycleAction call on its first terminating instance.
+        const asgArnWildcard = `arn:${stack.partition}:autoscaling:${stack.region}:${stack.account}:autoScalingGroup:*:autoScalingGroupName/*`;
+        const asgPolicy = new PolicyStatement({
+            effect: Effect.ALLOW,
+            actions: ["autoscaling:CompleteLifecycleAction"],
+            resources: [asgArnWildcard]
+        });
+        const volumeArnPattern = `arn:${stack.partition}:ec2:${stack.region}:${stack.account}:volume/*`;
+        const instanceArnPattern = `arn:${stack.partition}:ec2:${stack.region}:${stack.account}:instance/*`;
+        const dataVolumePolicies = dataVolumeOwnerLogicalId !== undefined
+            ? [
+                new PolicyStatement({
+                    effect: Effect.ALLOW,
+                    actions: ["ec2:DescribeVolumes"],
+                    resources: ["*"]
+                }),
+                new PolicyStatement({
+                    effect: Effect.ALLOW,
+                    actions: ["ec2:DetachVolume"],
+                    resources: [volumeArnPattern],
+                    conditions: {
+                        StringEquals: {
+                            [`aws:ResourceTag/${PERSISTENT_DATA_VOLUME_TAG_LIFECYCLE}`]: PERSISTENT_DATA_VOLUME_TAG_LIFECYCLE_VALUE,
+                            [`aws:ResourceTag/${PERSISTENT_DATA_VOLUME_TAG_STACK_ID}`]: Aws.STACK_ID
+                        }
+                    }
+                }),
+                new PolicyStatement({
+                    effect: Effect.ALLOW,
+                    actions: ["ec2:DetachVolume"],
+                    resources: [instanceArnPattern]
+                })
+            ]
+            : [];
+        this.lambda = new LambdaFunction(this, `${id}Fn`, {
+            runtime: Runtime.NODEJS_22_X,
+            handler: "index.handler",
+            code: Code.fromInline(source),
+            lambdaDescription: `${id} ${EC2_GRACEFUL_TERMINATION_HANDLER_DESCRIPTION}`,
+            roleDescription: `Execution role for ${id} ${EC2_GRACEFUL_TERMINATION_HANDLER_DESCRIPTION}`,
+            timeout: LAMBDA_TIMEOUT_SECONDS,
+            memorySize: 256,
+            logGroupRetention: RetentionDays.ONE_MONTH,
+            environment: {
+                ...(ecsClusterArn !== undefined && { ECS_CLUSTER_ARN: ecsClusterArn }),
+                ...(dataVolumeOwnerLogicalId !== undefined && {
+                    DATA_VOLUME_OWNER_LOGICAL_ID: dataVolumeOwnerLogicalId,
+                    DATA_VOLUME_STACK_ID: Aws.STACK_ID
+                })
+            },
+            inlinePolicy: [
+                ...ecsPolicies,
+                ec2Policies,
+                elbReadPolicy,
+                elbDeregisterPolicy,
+                asgPolicy,
+                ...dataVolumePolicies
+            ]
+        });
+        this.lambda.addSqsEventSource(this.queue.getQueue());
+        // Hook name embeds Aws.STACK_NAME (pseudo-parameter Ref, NOT a resource
+        // Ref — no CFN dep) so LifecycleHookName alone discriminates across
+        // stacks in the EB pattern. Sibling LAUNCHING hook uses "-launching".
+        const terminatingHookName = `${Aws.STACK_NAME}-${id}-terminating`;
+        attachInlineAsgLifecycleHook(this, `${id}TerminatingHook`, {
+            autoScalingGroup: props.autoScalingGroup,
+            hookName: terminatingHookName,
+            lifecycleTransition: LifecycleTransition.INSTANCE_TERMINATING,
+            defaultResult: DefaultResult.CONTINUE,
+            heartbeatTimeoutSeconds: HOOK_HEARTBEAT_SECONDS
+        });
+        // No AutoScalingGroupName in the pattern — that would create a CFN Ref to
+        // the ASG and block the EB rule on ASG CREATE_COMPLETE (deadlock — see
+        // wildcard ARN comment above and the PersistentDataVolume sibling).
+        new Subscription(this, `${id}TerminatingSub`, {
+            pattern: {
+                source: ["aws.autoscaling"],
+                detailType: ["EC2 Instance-terminate Lifecycle Action"],
+                detail: {
+                    LifecycleHookName: [terminatingHookName]
+                }
+            },
+            target: this.queue
+        });
+    }
+}
+function resolveOptionalString(value) {
+    if (value === undefined)
+        return undefined;
+    if (value === "")
+        return undefined;
+    return value;
+}

package/dist/lib/resources/aws/compute/ec2GracefulTerminationLambda.source.cjs

@@ -0,0 +1,458 @@
+/**
+ * EC2 graceful termination handler. Inlined into a Lambda at synth time by
+ * ec2GracefulTerminationHandler.ts. Triggered by an SQS queue fed by an
+ * EventBridge rule subscribed to the ASG EC2_INSTANCE_TERMINATING lifecycle
+ * event on the default bus.
+ *
+ * Wire format: the SQS message body is the EventBridge event envelope
+ *   `{ version, "detail-type", source, account, time, region, resources,
+ *      detail: { LifecycleActionToken, AutoScalingGroupName,
+ *                LifecycleHookName, EC2InstanceId, LifecycleTransition,
+ *                NotificationMetadata, ... } }`.
+ * The lifecycle payload lives at `body.detail`. Records whose envelope does
+ * not match (wrong source, missing detail, non-Lifecycle Action detail-type)
+ * are logged and skipped.
+ *
+ * Behaviour, in order:
+ *   1. (Conditional) ECS_CLUSTER_ARN set + non-empty → drain container
+ *      instance, wait for runningTasksCount=0, deregister.
+ *   2. (Always) Dissociate EIPs, deregister from any target groups,
+ *      detach non-primary ENIs.
+ *   3. CompleteLifecycleAction CONTINUE — even on partial failure, so the
+ *      ASG never stalls indefinitely. Better to terminate dirty than hang
+ *      the whole stack delete.
+ *
+ * CommonJS rather than ESM because Code.fromInline lands the source as
+ * `index.js`, which Lambda treats as CommonJS by default.
+ */
+const {
+  ECSClient,
+  ListContainerInstancesCommand,
+  DescribeContainerInstancesCommand,
+  UpdateContainerInstancesStateCommand,
+  DeregisterContainerInstanceCommand
+} = require("@aws-sdk/client-ecs");
+const {
+  EC2Client,
+  DescribeAddressesCommand,
+  DisassociateAddressCommand,
+  DescribeNetworkInterfacesCommand,
+  DetachNetworkInterfaceCommand,
+  DescribeVolumesCommand,
+  DetachVolumeCommand
+} = require("@aws-sdk/client-ec2");
+const {
+  ElasticLoadBalancingV2Client,
+  DescribeTargetGroupsCommand,
+  DescribeTargetHealthCommand,
+  DeregisterTargetsCommand
+} = require("@aws-sdk/client-elastic-load-balancing-v2");
+const {
+  AutoScalingClient,
+  CompleteLifecycleActionCommand
+} = require("@aws-sdk/client-auto-scaling");
+
+const DRAIN_POLL_INTERVAL_MS = 10_000;
+const DRAIN_DEADLINE_MS = 240_000;
+const DETACH_POLL_INTERVAL_MS = 5_000;
+const DETACH_DEADLINE_MS = 60_000;
+const TAG_OWNER_LOGICAL_ID = "fjall:OwnerLogicalId";
+const TAG_STACK_ID = "fjall:StackId";
+
+function errMessage(err) {
+  return err && err.message ? err.message : String(err);
+}
+
+let _ecs;
+let _ec2;
+let _elbv2;
+let _asg;
+function getEcs() {
+  if (!_ecs) _ecs = new ECSClient({});
+  return _ecs;
+}
+function getEc2() {
+  if (!_ec2) _ec2 = new EC2Client({});
+  return _ec2;
+}
+function getElbv2() {
+  if (!_elbv2) _elbv2 = new ElasticLoadBalancingV2Client({});
+  return _elbv2;
+}
+function getAsg() {
+  if (!_asg) _asg = new AutoScalingClient({});
+  return _asg;
+}
+
+function parseLifecycleMessage(record) {
+  const envelope = JSON.parse(record.body);
+  const detail = envelope && envelope.detail;
+  const source = envelope && envelope.source;
+  const detailType = envelope && envelope["detail-type"];
+  const valid =
+    detail !== undefined &&
+    detail !== null &&
+    typeof detail === "object" &&
+    source === "aws.autoscaling" &&
+    typeof detailType === "string" &&
+    detailType.endsWith("Lifecycle Action");
+  if (!valid) {
+    return { valid: false };
+  }
+  return {
+    valid: true,
+    instanceId: detail.EC2InstanceId,
+    asgName: detail.AutoScalingGroupName,
+    actionToken: detail.LifecycleActionToken,
+    hookName: detail.LifecycleHookName,
+    isTest: false
+  };
+}
+
+async function findContainerInstanceArn(ecsClient, clusterArn, instanceId) {
+  const list = await ecsClient.send(
+    new ListContainerInstancesCommand({
+      cluster: clusterArn,
+      filter: `ec2InstanceId == ${instanceId}`
+    })
+  );
+  if (!list.containerInstanceArns || list.containerInstanceArns.length === 0) {
+    return undefined;
+  }
+  return list.containerInstanceArns[0];
+}
+
+async function drainContainerInstance(
+  ecsClient,
+  clusterArn,
+  containerInstanceArn,
+  sleepFn,
+  nowFn
+) {
+  await ecsClient.send(
+    new UpdateContainerInstancesStateCommand({
+      cluster: clusterArn,
+      containerInstances: [containerInstanceArn],
+      status: "DRAINING"
+    })
+  );
+
+  const deadline = nowFn() + DRAIN_DEADLINE_MS;
+  while (nowFn() < deadline) {
+    const desc = await ecsClient.send(
+      new DescribeContainerInstancesCommand({
+        cluster: clusterArn,
+        containerInstances: [containerInstanceArn]
+      })
+    );
+    const instance =
+      desc.containerInstances && desc.containerInstances[0]
+        ? desc.containerInstances[0]
+        : undefined;
+    if (!instance) return;
+    if ((instance.runningTasksCount || 0) === 0) return;
+    await sleepFn(DRAIN_POLL_INTERVAL_MS);
+  }
+}
+
+async function dissociateInstanceEips(ec2Client, instanceId) {
+  const result = await ec2Client.send(
+    new DescribeAddressesCommand({
+      Filters: [{ Name: "instance-id", Values: [instanceId] }]
+    })
+  );
+  const addresses = result.Addresses || [];
+  for (const addr of addresses) {
+    if (!addr.AssociationId) continue;
+    await ec2Client.send(
+      new DisassociateAddressCommand({ AssociationId: addr.AssociationId })
+    );
+  }
+}
+
+async function deregisterFromTargetGroups(elbv2Client, instanceId) {
+  const groups = await elbv2Client.send(new DescribeTargetGroupsCommand({}));
+  const targetGroups = groups.TargetGroups || [];
+  for (const tg of targetGroups) {
+    if (!tg.TargetGroupArn) continue;
+    let health;
+    try {
+      health = await elbv2Client.send(
+        new DescribeTargetHealthCommand({ TargetGroupArn: tg.TargetGroupArn })
+      );
+    } catch (err) {
+      console.debug(
+        JSON.stringify({
+          message: "DescribeTargetHealth skipped",
+          targetGroupArn: tg.TargetGroupArn,
+          error: errMessage(err)
+        })
+      );
+      continue;
+    }
+    const matches = (health.TargetHealthDescriptions || []).filter(
+      (d) => d.Target && d.Target.Id === instanceId
+    );
+    if (matches.length === 0) continue;
+    await elbv2Client.send(
+      new DeregisterTargetsCommand({
+        TargetGroupArn: tg.TargetGroupArn,
+        Targets: matches.map((m) => ({
+          Id: m.Target.Id,
+          Port: m.Target.Port
+        }))
+      })
+    );
+  }
+}
+
+async function detachSecondaryEnis(ec2Client, instanceId) {
+  const result = await ec2Client.send(
+    new DescribeNetworkInterfacesCommand({
+      Filters: [{ Name: "attachment.instance-id", Values: [instanceId] }]
+    })
+  );
+  const enis = result.NetworkInterfaces || [];
+  for (const eni of enis) {
+    const attachment = eni.Attachment;
+    if (!attachment || !attachment.AttachmentId) continue;
+    // Skip the primary interface (DeviceIndex 0); it is destroyed with the
+    // instance and cannot be detached independently.
+    if (attachment.DeviceIndex === 0) continue;
+    try {
+      await ec2Client.send(
+        new DetachNetworkInterfaceCommand({
+          AttachmentId: attachment.AttachmentId,
+          Force: true
+        })
+      );
+    } catch (err) {
+      console.debug(
+        JSON.stringify({
+          message: "DetachNetworkInterface skipped",
+          attachmentId: attachment.AttachmentId,
+          error: errMessage(err)
+        })
+      );
+    }
+  }
+}
+
+async function detachDataVolume(
+  ec2Client,
+  instanceId,
+  ownerLogicalId,
+  stackId,
+  sleepFn,
+  nowFn
+) {
+  const described = await ec2Client.send(
+    new DescribeVolumesCommand({
+      Filters: [
+        { Name: `tag:${TAG_OWNER_LOGICAL_ID}`, Values: [ownerLogicalId] },
+        { Name: `tag:${TAG_STACK_ID}`, Values: [stackId] },
+        { Name: "attachment.instance-id", Values: [instanceId] }
+      ]
+    })
+  );
+  const volumes = described.Volumes || [];
+  if (volumes.length === 0) return;
+  const volumeId = volumes[0].VolumeId;
+  if (!volumeId) return;
+  await ec2Client.send(
+    new DetachVolumeCommand({ VolumeId: volumeId, InstanceId: instanceId })
+  );
+  const deadline = nowFn() + DETACH_DEADLINE_MS;
+  while (nowFn() < deadline) {
+    const desc = await ec2Client.send(
+      new DescribeVolumesCommand({ VolumeIds: [volumeId] })
+    );
+    const volume =
+      desc.Volumes && desc.Volumes[0] ? desc.Volumes[0] : undefined;
+    if (volume && volume.State === "available") return;
+    await sleepFn(DETACH_POLL_INTERVAL_MS);
+  }
+}
+
+async function completeLifecycle(asgClient, asgName, hookName, actionToken) {
+  await asgClient.send(
+    new CompleteLifecycleActionCommand({
+      AutoScalingGroupName: asgName,
+      LifecycleHookName: hookName,
+      LifecycleActionToken: actionToken,
+      LifecycleActionResult: "CONTINUE"
+    })
+  );
+}
+
+function readEnv(env, name) {
+  const value = env[name];
+  if (typeof value !== "string" || value === "") return undefined;
+  return value;
+}
+
+async function processRecord(record, deps) {
+  const ecsClient = (deps && deps.ecsClient) || getEcs();
+  const ec2Client = (deps && deps.ec2Client) || getEc2();
+  const elbv2Client = (deps && deps.elbv2Client) || getElbv2();
+  const asgClient = (deps && deps.asgClient) || getAsg();
+  const sleepFn =
+    (deps && deps.sleep) ||
+    ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+  const nowFn = (deps && deps.now) || (() => Date.now());
+  const env = (deps && deps.env) || process.env;
+  const log =
+    (deps && deps.log) ||
+    ((msg, fields) => {
+      console.log(JSON.stringify({ message: msg, ...(fields || {}) }));
+    });
+
+  const parsed = parseLifecycleMessage(record);
+  if (!parsed.valid) {
+    log("Record is not an aws.autoscaling Lifecycle Action; skipping", {
+      bodyPreview: record.body ? String(record.body).slice(0, 200) : undefined
+    });
+    return;
+  }
+  const { instanceId, asgName, actionToken, hookName } = parsed;
+
+  const clusterArn = readEnv(env, "ECS_CLUSTER_ARN");
+  if (clusterArn !== undefined) {
+    try {
+      const containerInstanceArn = await findContainerInstanceArn(
+        ecsClient,
+        clusterArn,
+        instanceId
+      );
+      if (containerInstanceArn !== undefined) {
+        await drainContainerInstance(
+          ecsClient,
+          clusterArn,
+          containerInstanceArn,
+          sleepFn,
+          nowFn
+        );
+        await ecsClient.send(
+          new DeregisterContainerInstanceCommand({
+            cluster: clusterArn,
+            containerInstance: containerInstanceArn,
+            force: true
+          })
+        );
+      }
+    } catch (err) {
+      log("ECS drain step failed; continuing to generic cleanup", {
+        instanceId,
+        error: errMessage(err)
+      });
+    }
+  }
+
+  try {
+    await dissociateInstanceEips(ec2Client, instanceId);
+  } catch (err) {
+    log("EIP dissociation failed", {
+      instanceId,
+      error: errMessage(err)
+    });
+  }
+  try {
+    await deregisterFromTargetGroups(elbv2Client, instanceId);
+  } catch (err) {
+    log("Target-group deregistration failed", {
+      instanceId,
+      error: errMessage(err)
+    });
+  }
+  try {
+    await detachSecondaryEnis(ec2Client, instanceId);
+  } catch (err) {
+    log("ENI detachment failed", {
+      instanceId,
+      error: errMessage(err)
+    });
+  }
+
+  const dataVolumeOwnerLogicalId = readEnv(env, "DATA_VOLUME_OWNER_LOGICAL_ID");
+  const dataVolumeStackId = readEnv(env, "DATA_VOLUME_STACK_ID");
+  if (
+    dataVolumeOwnerLogicalId !== undefined &&
+    dataVolumeStackId !== undefined
+  ) {
+    try {
+      await detachDataVolume(
+        ec2Client,
+        instanceId,
+        dataVolumeOwnerLogicalId,
+        dataVolumeStackId,
+        sleepFn,
+        nowFn
+      );
+    } catch (err) {
+      log("Data volume detach failed; AWS will force-detach after heartbeat", {
+        instanceId,
+        ownerLogicalId: dataVolumeOwnerLogicalId,
+        error: errMessage(err)
+      });
+    }
+  }
+
+  await completeLifecycle(asgClient, asgName, hookName, actionToken);
+}
+
+exports.handler = async (event) => {
+  const records = (event && event.Records) || [];
+  for (const record of records) {
+    try {
+      await processRecord(record);
+    } catch (err) {
+      // Last-resort: try to release the lifecycle action so the stack does
+      // not hang. Failure here is logged and the message will retry via SQS.
+      console.error(
+        JSON.stringify({
+          message: "Graceful termination handler crashed; attempting CONTINUE",
+          error: errMessage(err)
+        })
+      );
+      try {
+        const parsed = parseLifecycleMessage(record);
+        if (!parsed.valid) {
+          continue;
+        }
+        await completeLifecycle(
+          getAsg(),
+          parsed.asgName,
+          parsed.hookName,
+          parsed.actionToken
+        );
+      } catch (innerErr) {
+        console.error(
+          JSON.stringify({
+            message: "CompleteLifecycleAction also failed",
+            error: errMessage(innerErr)
+          })
+        );
+        throw err;
+      }
+    }
+  }
+};
+
+exports._internals = {
+  parseLifecycleMessage,
+  findContainerInstanceArn,
+  drainContainerInstance,
+  dissociateInstanceEips,
+  deregisterFromTargetGroups,
+  detachSecondaryEnis,
+  detachDataVolume,
+  completeLifecycle,
+  readEnv,
+  processRecord,
+  DRAIN_POLL_INTERVAL_MS,
+  DRAIN_DEADLINE_MS,
+  DETACH_POLL_INTERVAL_MS,
+  DETACH_DEADLINE_MS,
+  TAG_OWNER_LOGICAL_ID,
+  TAG_STACK_ID
+};