@fjall/components-infrastructure 39.0.0 → 39.0.2

package/dist/lib/resources/aws/compute/ec2GracefulTerminationLambda.source.cjs

@@ -14,13 +14,35 @@
  * are logged and skipped.
  *
  * Behaviour, in order:
- *   1. (Conditional) ECS_CLUSTER_ARN set + non-empty → drain container
- *      instance, wait for runningTasksCount=0, deregister.
+ *   1. (Conditional) ECS_CLUSTER_ARN set + non-empty → set the container
+ *      instance DRAINING and wait until every task on it is proven STOPPED;
+ *      a task still short of STOPPED at the drain deadline is sent StopTask
+ *      and waited on again. Then deregister the container instance. The
+ *      outcome is logged as `event: "drain_proven"` or
+ *      `event: "drain_unproven"`; the construct alarms on the second.
  *   2. (Always) Dissociate EIPs, deregister from any target groups,
  *      detach non-primary ENIs.
  *   3. CompleteLifecycleAction CONTINUE — even on partial failure, so the
  *      ASG never stalls indefinitely. Better to terminate dirty than hang
- *      the whole stack delete.
+ *      the whole stack delete. A token whose action has already ended is
+ *      logged as `event: "lifecycle_action_already_ended"`, not retried.
+ *
+ * STOPPED (or DELETED, which ECS reports after it) is the only proof. A
+ * container instance's `runningTasksCount` drops a task the moment it leaves
+ * RUNNING, while the task is still STOPPING and its container is still
+ * writing; reading that count as "the instance is idle" is how a data volume
+ * was once detached from under a running ClickHouse.
+ *
+ * A proof counts only if it was read before the lifecycle hook's heartbeat
+ * (LIFECYCLE_HEARTBEAT_SECONDS, dated from the event's time) could have let
+ * the Auto Scaling group terminate the instance on its own. A later proof is
+ * logged as unproven with `provenAfterHeartbeat: true`.
+ *
+ * This handler never detaches an EBS volume. EC2 requires a volume to be
+ * unmounted before it is detached, and nothing here can unmount it: the
+ * instance's own shutdown unmounts every filesystem, and termination then
+ * releases the volume for the PersistentDataVolume LAUNCHING handler, which
+ * waits for that release before it attaches.
  *
  * CommonJS rather than ESM because Code.fromInline lands the source as
  * `index.js`, which Lambda treats as CommonJS by default.
@@ -28,18 +50,18 @@
 const {
   ECSClient,
   ListContainerInstancesCommand,
-  DescribeContainerInstancesCommand,
   UpdateContainerInstancesStateCommand,
-  DeregisterContainerInstanceCommand
+  DeregisterContainerInstanceCommand,
+  ListTasksCommand,
+  DescribeTasksCommand,
+  StopTaskCommand
 } = require("@aws-sdk/client-ecs");
 const {
   EC2Client,
   DescribeAddressesCommand,
   DisassociateAddressCommand,
   DescribeNetworkInterfacesCommand,
-  DetachNetworkInterfaceCommand,
-  DescribeVolumesCommand,
-  DetachVolumeCommand
+  DetachNetworkInterfaceCommand
 } = require("@aws-sdk/client-ec2");
 const {
   ElasticLoadBalancingV2Client,
@@ -53,16 +75,54 @@ const {
 } = require("@aws-sdk/client-auto-scaling");
 
 const DRAIN_POLL_INTERVAL_MS = 10_000;
+// How long DRAINING is given to stop the instance's tasks on its own. The
+// service scheduler stops service tasks; standalone tasks run to completion.
 const DRAIN_DEADLINE_MS = 240_000;
-const DETACH_POLL_INTERVAL_MS = 5_000;
-const DETACH_DEADLINE_MS = 60_000;
-const TAG_OWNER_LOGICAL_ID = "fjall:OwnerLogicalId";
-const TAG_STACK_ID = "fjall:StackId";
+// How long a task sent StopTask is given to reach STOPPED. ECS caps a
+// container's stopTimeout at 120 s before it sends SIGKILL.
+const STOP_DEADLINE_MS = 150_000;
+// DescribeTasks accepts at most 100 task ARNs per call.
+const DESCRIBE_TASKS_BATCH_SIZE = 100;
+// ListTasks never reports a desired status of PENDING, so these two cover
+// every task ECS still tracks on the instance.
+const TASK_DESIRED_STATUSES = ["RUNNING", "STOPPED"];
+// ECS reports DELETED after STOPPED ("a transition state when a task stops",
+// shown only by DescribeTasks). Either means the task's containers have
+// exited. Any other status, including one ECS adds later, is not proof.
+const STOPPED_TASK_STATUSES = ["STOPPED", "DELETED"];
+// Bounds every AWS call, SDK retries included, so a hung request cannot carry
+// an attempt past its deadlines.
+const SDK_CALL_TIMEOUT_MS = 20_000;
+// Kept back from the end of the invocation for deregistration, the EIP,
+// target-group and ENI cleanup, and CompleteLifecycleAction.
+const COMPLETION_RESERVE_MS = 90_000;
+// Kept back from the end of the lifecycle hook's heartbeat. The event's time
+// is when the action began, so a proof read after this point may postdate
+// the Auto Scaling group releasing the instance on its own.
+const HEARTBEAT_RESERVE_MS = 30_000;
+const DRAIN_EVENT = {
+  PROVEN: "drain_proven",
+  UNPROVEN: "drain_unproven",
+  NOT_REGISTERED: "drain_not_registered"
+};
+const LIFECYCLE_EVENT = {
+  ALREADY_ENDED: "lifecycle_action_already_ended"
+};
+const LIFECYCLE_OUTCOME = {
+  COMPLETED: "completed",
+  ALREADY_ENDED: "already_ended"
+};
 
 function errMessage(err) {
   return err && err.message ? err.message : String(err);
 }
 
+function send(client, command) {
+  return client.send(command, {
+    abortSignal: AbortSignal.timeout(SDK_CALL_TIMEOUT_MS)
+  });
+}
+
 let _ecs;
 let _ec2;
 let _elbv2;
@@ -105,12 +165,48 @@ function parseLifecycleMessage(record) {
     asgName: detail.AutoScalingGroupName,
     actionToken: detail.LifecycleActionToken,
     hookName: detail.LifecycleHookName,
+    startedAtMs: lifecycleActionStartedAtMs(record, envelope),
     isTest: false
   };
 }
 
+// When the lifecycle action began, as near as the message can say: the
+// event's own time, else the moment EventBridge sent it to the queue.
+function lifecycleActionStartedAtMs(record, envelope) {
+  const eventTime =
+    typeof envelope.time === "string" ? Date.parse(envelope.time) : NaN;
+  if (Number.isFinite(eventTime)) return eventTime;
+  const sentAt = Number(record.attributes && record.attributes.SentTimestamp);
+  return Number.isFinite(sentAt) && sentAt > 0 ? sentAt : undefined;
+}
+
+// The attempt's two deadlines, on the Lambda's clock. A proof read after
+// `proofDeadlineMs` is not trusted; no drain phase runs past
+// `workDeadlineMs`, which also leaves the invocation time to complete.
+function attemptDeadlines(startedAtMs, heartbeatSeconds, invocationEndMs) {
+  const proofDeadlineMs =
+    startedAtMs !== undefined && heartbeatSeconds !== undefined
+      ? startedAtMs + heartbeatSeconds * 1000 - HEARTBEAT_RESERVE_MS
+      : Infinity;
+  const invocationWorkEndMs =
+    invocationEndMs !== undefined
+      ? invocationEndMs - COMPLETION_RESERVE_MS
+      : Infinity;
+  return {
+    proofDeadlineMs,
+    workDeadlineMs: Math.min(proofDeadlineMs, invocationWorkEndMs)
+  };
+}
+
+function readPositiveInteger(env, name) {
+  const value = readEnv(env, name);
+  if (value === undefined || !/^[1-9][0-9]*$/.test(value)) return undefined;
+  return Number.parseInt(value, 10);
+}
+
 async function findContainerInstanceArn(ecsClient, clusterArn, instanceId) {
-  const list = await ecsClient.send(
+  const list = await send(
+    ecsClient,
     new ListContainerInstancesCommand({
       cluster: clusterArn,
       filter: `ec2InstanceId == ${instanceId}`
@@ -122,41 +218,224 @@ async function findContainerInstanceArn(ecsClient, clusterArn, instanceId) {
   return list.containerInstanceArns[0];
 }
 
-async function drainContainerInstance(
-  ecsClient,
-  clusterArn,
+async function listInstanceTaskArns(ctx, containerInstanceArn) {
+  const taskArns = new Set();
+  for (const desiredStatus of TASK_DESIRED_STATUSES) {
+    let nextToken;
+    do {
+      const page = await send(
+        ctx.ecsClient,
+        new ListTasksCommand({
+          cluster: ctx.clusterArn,
+          containerInstance: containerInstanceArn,
+          desiredStatus,
+          ...(nextToken !== undefined && { nextToken })
+        })
+      );
+      for (const taskArn of page.taskArns || []) taskArns.add(taskArn);
+      nextToken = page.nextToken;
+    } while (nextToken);
+  }
+  return [...taskArns];
+}
+
+// The ARNs of the instance's tasks not proven STOPPED. A task DescribeTasks
+// cannot describe (a `failures` entry) is not proven either; once ECS has
+// forgotten it, the next ListTasks no longer names it.
+async function unstoppedTaskArns(ctx, containerInstanceArn) {
+  const taskArns = await listInstanceTaskArns(ctx, containerInstanceArn);
+  const unstopped = [];
+  for (let i = 0; i < taskArns.length; i += DESCRIBE_TASKS_BATCH_SIZE) {
+    const batch = taskArns.slice(i, i + DESCRIBE_TASKS_BATCH_SIZE);
+    const described = await send(
+      ctx.ecsClient,
+      new DescribeTasksCommand({ cluster: ctx.clusterArn, tasks: batch })
+    );
+    const stopped = new Set(
+      (described.tasks || [])
+        .filter((task) => STOPPED_TASK_STATUSES.includes(task.lastStatus))
+        .map((task) => task.taskArn)
+    );
+    for (const taskArn of batch) {
+      if (!stopped.has(taskArn)) unstopped.push(taskArn);
+    }
+  }
+  return unstopped;
+}
+
+// Polls until every task is STOPPED or the budget is spent, reading once
+// more at the deadline, so the last read starts no later than it. The budget never runs past the attempt's own
+// deadline. A failed read is retried, and a verdict whose last read failed
+// is unproven. A proof read after the proof deadline is unproven too, since
+// the instance may already have been released. `onUnstopped` sees each
+// successful read's list.
+async function awaitTasksStopped(
+  ctx,
   containerInstanceArn,
-  sleepFn,
-  nowFn
+  budgetMs,
+  onUnstopped
 ) {
-  await ecsClient.send(
+  const deadline = Math.min(ctx.now() + budgetMs, ctx.workDeadlineMs);
+  for (;;) {
+    let verdict;
+    try {
+      const unstopped = await unstoppedTaskArns(ctx, containerInstanceArn);
+      verdict = { proven: unstopped.length === 0, unstopped };
+    } catch (err) {
+      ctx.log("Task status read failed; retrying until the deadline", {
+        containerInstanceArn,
+        error: errMessage(err)
+      });
+      verdict = { proven: false, unstopped: undefined, error: errMessage(err) };
+    }
+    if (verdict.proven && ctx.now() > ctx.proofDeadlineMs) {
+      return { proven: false, unstopped: [], provenAfterHeartbeat: true };
+    }
+    if (verdict.proven || ctx.now() >= deadline) return verdict;
+    if (verdict.unstopped !== undefined) await onUnstopped(verdict.unstopped);
+    await ctx.sleep(
+      Math.max(0, Math.min(DRAIN_POLL_INTERVAL_MS, deadline - ctx.now()))
+    );
+  }
+}
+
+async function drainContainerInstance(ctx, instanceId, containerInstanceArn) {
+  await send(
+    ctx.ecsClient,
     new UpdateContainerInstancesStateCommand({
-      cluster: clusterArn,
+      cluster: ctx.clusterArn,
       containerInstances: [containerInstanceArn],
       status: "DRAINING"
     })
   );
 
-  const deadline = nowFn() + DRAIN_DEADLINE_MS;
-  while (nowFn() < deadline) {
-    const desc = await ecsClient.send(
-      new DescribeContainerInstancesCommand({
-        cluster: clusterArn,
-        containerInstances: [containerInstanceArn]
+  const drained = await awaitTasksStopped(
+    ctx,
+    containerInstanceArn,
+    DRAIN_DEADLINE_MS,
+    async () => {}
+  );
+  if (
+    drained.proven ||
+    drained.provenAfterHeartbeat ||
+    ctx.now() >= ctx.workDeadlineMs
+  ) {
+    return { ...drained, stopRequested: [] };
+  }
+
+  const stopRequested = new Set();
+  const stopped = await awaitTasksStopped(
+    ctx,
+    containerInstanceArn,
+    STOP_DEADLINE_MS,
+    async (unstopped) => {
+      for (const taskArn of unstopped) {
+        if (stopRequested.has(taskArn)) continue;
+        try {
+          await send(
+            ctx.ecsClient,
+            new StopTaskCommand({
+              cluster: ctx.clusterArn,
+              task: taskArn,
+              reason: `Instance ${instanceId} is terminating and the task did not stop within the drain deadline`
+            })
+          );
+          stopRequested.add(taskArn);
+        } catch (err) {
+          ctx.log("StopTask failed; retrying on the next poll", {
+            taskArn,
+            error: errMessage(err)
+          });
+        }
+      }
+    }
+  );
+  return { ...stopped, stopRequested: [...stopRequested] };
+}
+
+function drainVerdictMessage(verdict) {
+  if (verdict.proven) return "Every task on the instance is STOPPED";
+  if (verdict.provenAfterHeartbeat) {
+    return "Every task on the instance is STOPPED, but only after the lifecycle hook's heartbeat could have released the instance";
+  }
+  return "Tasks on the instance are not proven STOPPED; terminating anyway";
+}
+
+// Drains, proves and deregisters the instance's container instance. Never
+// throws: every failure is logged as an unproven drain, because the
+// lifecycle action must still complete.
+async function drainEcsTasks(ctx, instanceId) {
+  let containerInstanceArn;
+  try {
+    containerInstanceArn = await findContainerInstanceArn(
+      ctx.ecsClient,
+      ctx.clusterArn,
+      instanceId
+    );
+  } catch (err) {
+    ctx.log("Container instance lookup failed; the drain is unproven", {
+      event: DRAIN_EVENT.UNPROVEN,
+      instanceId,
+      error: errMessage(err)
+    });
+    return;
+  }
+  if (containerInstanceArn === undefined) {
+    ctx.log("Instance is not registered in the cluster; nothing to drain", {
+      event: DRAIN_EVENT.NOT_REGISTERED,
+      instanceId
+    });
+    return;
+  }
+
+  try {
+    const verdict = await drainContainerInstance(
+      ctx,
+      instanceId,
+      containerInstanceArn
+    );
+    ctx.log(drainVerdictMessage(verdict), {
+      event: verdict.proven ? DRAIN_EVENT.PROVEN : DRAIN_EVENT.UNPROVEN,
+      instanceId,
+      containerInstanceArn,
+      stopRequestedTaskArns: verdict.stopRequested,
+      ...(verdict.provenAfterHeartbeat && { provenAfterHeartbeat: true }),
+      ...(!verdict.proven &&
+        !verdict.provenAfterHeartbeat && {
+          unstoppedTaskArns: verdict.unstopped,
+          ...(verdict.error !== undefined && { error: verdict.error })
+        })
+    });
+  } catch (err) {
+    ctx.log("ECS drain failed; the drain is unproven", {
+      event: DRAIN_EVENT.UNPROVEN,
+      instanceId,
+      containerInstanceArn,
+      error: errMessage(err)
+    });
+  }
+
+  try {
+    await send(
+      ctx.ecsClient,
+      new DeregisterContainerInstanceCommand({
+        cluster: ctx.clusterArn,
+        containerInstance: containerInstanceArn,
+        force: true
       })
     );
-    const instance =
-      desc.containerInstances && desc.containerInstances[0]
-        ? desc.containerInstances[0]
-        : undefined;
-    if (!instance) return;
-    if ((instance.runningTasksCount || 0) === 0) return;
-    await sleepFn(DRAIN_POLL_INTERVAL_MS);
+  } catch (err) {
+    ctx.log("Container instance deregistration failed", {
+      instanceId,
+      containerInstanceArn,
+      error: errMessage(err)
+    });
   }
 }
 
 async function dissociateInstanceEips(ec2Client, instanceId) {
-  const result = await ec2Client.send(
+  const result = await send(
+    ec2Client,
     new DescribeAddressesCommand({
       Filters: [{ Name: "instance-id", Values: [instanceId] }]
     })
@@ -164,20 +443,22 @@ async function dissociateInstanceEips(ec2Client, instanceId) {
   const addresses = result.Addresses || [];
   for (const addr of addresses) {
     if (!addr.AssociationId) continue;
-    await ec2Client.send(
+    await send(
+      ec2Client,
       new DisassociateAddressCommand({ AssociationId: addr.AssociationId })
     );
   }
 }
 
 async function deregisterFromTargetGroups(elbv2Client, instanceId) {
-  const groups = await elbv2Client.send(new DescribeTargetGroupsCommand({}));
+  const groups = await send(elbv2Client, new DescribeTargetGroupsCommand({}));
   const targetGroups = groups.TargetGroups || [];
   for (const tg of targetGroups) {
     if (!tg.TargetGroupArn) continue;
     let health;
     try {
-      health = await elbv2Client.send(
+      health = await send(
+        elbv2Client,
         new DescribeTargetHealthCommand({ TargetGroupArn: tg.TargetGroupArn })
       );
     } catch (err) {
@@ -194,7 +475,8 @@ async function deregisterFromTargetGroups(elbv2Client, instanceId) {
       (d) => d.Target && d.Target.Id === instanceId
     );
     if (matches.length === 0) continue;
-    await elbv2Client.send(
+    await send(
+      elbv2Client,
       new DeregisterTargetsCommand({
         TargetGroupArn: tg.TargetGroupArn,
         Targets: matches.map((m) => ({
@@ -207,7 +489,8 @@ async function deregisterFromTargetGroups(elbv2Client, instanceId) {
 }
 
 async function detachSecondaryEnis(ec2Client, instanceId) {
-  const result = await ec2Client.send(
+  const result = await send(
+    ec2Client,
     new DescribeNetworkInterfacesCommand({
       Filters: [{ Name: "attachment.instance-id", Values: [instanceId] }]
     })
@@ -220,7 +503,8 @@ async function detachSecondaryEnis(ec2Client, instanceId) {
     // instance and cannot be detached independently.
     if (attachment.DeviceIndex === 0) continue;
     try {
-      await ec2Client.send(
+      await send(
+        ec2Client,
         new DetachNetworkInterfaceCommand({
           AttachmentId: attachment.AttachmentId,
           Force: true
@@ -238,50 +522,40 @@ async function detachSecondaryEnis(ec2Client, instanceId) {
   }
 }
 
-async function detachDataVolume(
-  ec2Client,
-  instanceId,
-  ownerLogicalId,
-  stackId,
-  sleepFn,
-  nowFn
-) {
-  const described = await ec2Client.send(
-    new DescribeVolumesCommand({
-      Filters: [
-        { Name: `tag:${TAG_OWNER_LOGICAL_ID}`, Values: [ownerLogicalId] },
-        { Name: `tag:${TAG_STACK_ID}`, Values: [stackId] },
-        { Name: "attachment.instance-id", Values: [instanceId] }
-      ]
-    })
+// Auto Scaling refuses a token whose action has already ended, because
+// another delivery of the event completed it or the heartbeat expired, with
+// a ValidationError naming no active lifecycle action.
+function isNoActiveLifecycleAction(err) {
+  return (
+    Boolean(err) &&
+    err.name === "ValidationError" &&
+    /No active Lifecycle Action/i.test(errMessage(err))
   );
-  const volumes = described.Volumes || [];
-  if (volumes.length === 0) return;
-  const volumeId = volumes[0].VolumeId;
-  if (!volumeId) return;
-  await ec2Client.send(
-    new DetachVolumeCommand({ VolumeId: volumeId, InstanceId: instanceId })
-  );
-  const deadline = nowFn() + DETACH_DEADLINE_MS;
-  while (nowFn() < deadline) {
-    const desc = await ec2Client.send(
-      new DescribeVolumesCommand({ VolumeIds: [volumeId] })
+}
+
+async function completeLifecycle(asgClient, asgName, hookName, actionToken) {
+  try {
+    await send(
+      asgClient,
+      new CompleteLifecycleActionCommand({
+        AutoScalingGroupName: asgName,
+        LifecycleHookName: hookName,
+        LifecycleActionToken: actionToken,
+        LifecycleActionResult: "CONTINUE"
+      })
     );
-    const volume =
-      desc.Volumes && desc.Volumes[0] ? desc.Volumes[0] : undefined;
-    if (volume && volume.State === "available") return;
-    await sleepFn(DETACH_POLL_INTERVAL_MS);
+    return LIFECYCLE_OUTCOME.COMPLETED;
+  } catch (err) {
+    if (isNoActiveLifecycleAction(err)) return LIFECYCLE_OUTCOME.ALREADY_ENDED;
+    throw err;
   }
 }
 
-async function completeLifecycle(asgClient, asgName, hookName, actionToken) {
-  await asgClient.send(
-    new CompleteLifecycleActionCommand({
-      AutoScalingGroupName: asgName,
-      LifecycleHookName: hookName,
-      LifecycleActionToken: actionToken,
-      LifecycleActionResult: "CONTINUE"
-    })
+function logLifecycleOutcome(log, outcome, instanceId) {
+  if (outcome !== LIFECYCLE_OUTCOME.ALREADY_ENDED) return;
+  log(
+    "The lifecycle action had already ended: another delivery completed it, or its heartbeat expired",
+    { event: LIFECYCLE_EVENT.ALREADY_ENDED, instanceId }
   );
 }
 
@@ -291,6 +565,10 @@ function readEnv(env, name) {
   return value;
 }
 
+function consoleLog(msg, fields) {
+  console.log(JSON.stringify({ message: msg, ...(fields || {}) }));
+}
+
 async function processRecord(record, deps) {
   const ecsClient = (deps && deps.ecsClient) || getEcs();
   const ec2Client = (deps && deps.ec2Client) || getEc2();
@@ -301,11 +579,7 @@ async function processRecord(record, deps) {
     ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
   const nowFn = (deps && deps.now) || (() => Date.now());
   const env = (deps && deps.env) || process.env;
-  const log =
-    (deps && deps.log) ||
-    ((msg, fields) => {
-      console.log(JSON.stringify({ message: msg, ...(fields || {}) }));
-    });
+  const log = (deps && deps.log) || consoleLog;
 
   const parsed = parseLifecycleMessage(record);
   if (!parsed.valid) {
@@ -318,34 +592,36 @@ async function processRecord(record, deps) {
 
   const clusterArn = readEnv(env, "ECS_CLUSTER_ARN");
   if (clusterArn !== undefined) {
-    try {
-      const containerInstanceArn = await findContainerInstanceArn(
-        ecsClient,
-        clusterArn,
-        instanceId
+    const heartbeatSeconds = readPositiveInteger(
+      env,
+      "LIFECYCLE_HEARTBEAT_SECONDS"
+    );
+    const deadlines = attemptDeadlines(
+      parsed.startedAtMs,
+      heartbeatSeconds,
+      deps && deps.invocationEndMs
+    );
+    if (deadlines.proofDeadlineMs === Infinity) {
+      log(
+        "The drain's proof cannot be dated against the lifecycle hook's heartbeat",
+        {
+          instanceId,
+          heartbeatSecondsKnown: heartbeatSeconds !== undefined,
+          startedAtKnown: parsed.startedAtMs !== undefined
+        }
       );
-      if (containerInstanceArn !== undefined) {
-        await drainContainerInstance(
-          ecsClient,
-          clusterArn,
-          containerInstanceArn,
-          sleepFn,
-          nowFn
-        );
-        await ecsClient.send(
-          new DeregisterContainerInstanceCommand({
-            cluster: clusterArn,
-            containerInstance: containerInstanceArn,
-            force: true
-          })
-        );
-      }
-    } catch (err) {
-      log("ECS drain step failed; continuing to generic cleanup", {
-        instanceId,
-        error: errMessage(err)
-      });
     }
+    await drainEcsTasks(
+      {
+        ecsClient,
+        clusterArn,
+        sleep: sleepFn,
+        now: nowFn,
+        log,
+        ...deadlines
+      },
+      instanceId
+    );
   }
 
   try {
@@ -373,38 +649,22 @@ async function processRecord(record, deps) {
     });
   }
 
-  const dataVolumeOwnerLogicalId = readEnv(env, "DATA_VOLUME_OWNER_LOGICAL_ID");
-  const dataVolumeStackId = readEnv(env, "DATA_VOLUME_STACK_ID");
-  if (
-    dataVolumeOwnerLogicalId !== undefined &&
-    dataVolumeStackId !== undefined
-  ) {
-    try {
-      await detachDataVolume(
-        ec2Client,
-        instanceId,
-        dataVolumeOwnerLogicalId,
-        dataVolumeStackId,
-        sleepFn,
-        nowFn
-      );
-    } catch (err) {
-      log("Data volume detach failed; AWS will force-detach after heartbeat", {
-        instanceId,
-        ownerLogicalId: dataVolumeOwnerLogicalId,
-        error: errMessage(err)
-      });
-    }
-  }
-
-  await completeLifecycle(asgClient, asgName, hookName, actionToken);
+  logLifecycleOutcome(
+    log,
+    await completeLifecycle(asgClient, asgName, hookName, actionToken),
+    instanceId
+  );
 }
 
-exports.handler = async (event) => {
+exports.handler = async (event, context) => {
+  const invocationEndMs =
+    context && typeof context.getRemainingTimeInMillis === "function"
+      ? Date.now() + context.getRemainingTimeInMillis()
+      : undefined;
   const records = (event && event.Records) || [];
   for (const record of records) {
     try {
-      await processRecord(record);
+      await processRecord(record, { invocationEndMs });
     } catch (err) {
       // Last-resort: try to release the lifecycle action so the stack does
       // not hang. Failure here is logged and the message will retry via SQS.
@@ -419,11 +679,15 @@ exports.handler = async (event) => {
         if (!parsed.valid) {
           continue;
         }
-        await completeLifecycle(
-          getAsg(),
-          parsed.asgName,
-          parsed.hookName,
-          parsed.actionToken
+        logLifecycleOutcome(
+          consoleLog,
+          await completeLifecycle(
+            getAsg(),
+            parsed.asgName,
+            parsed.hookName,
+            parsed.actionToken
+          ),
+          parsed.instanceId
         );
       } catch (innerErr) {
         console.error(
@@ -440,19 +704,30 @@ exports.handler = async (event) => {
 
 exports._internals = {
   parseLifecycleMessage,
+  attemptDeadlines,
   findContainerInstanceArn,
+  listInstanceTaskArns,
+  unstoppedTaskArns,
+  awaitTasksStopped,
   drainContainerInstance,
+  drainEcsTasks,
   dissociateInstanceEips,
   deregisterFromTargetGroups,
   detachSecondaryEnis,
-  detachDataVolume,
+  isNoActiveLifecycleAction,
   completeLifecycle,
   readEnv,
   processRecord,
   DRAIN_POLL_INTERVAL_MS,
   DRAIN_DEADLINE_MS,
-  DETACH_POLL_INTERVAL_MS,
-  DETACH_DEADLINE_MS,
-  TAG_OWNER_LOGICAL_ID,
-  TAG_STACK_ID
+  STOP_DEADLINE_MS,
+  DESCRIBE_TASKS_BATCH_SIZE,
+  TASK_DESIRED_STATUSES,
+  STOPPED_TASK_STATUSES,
+  SDK_CALL_TIMEOUT_MS,
+  COMPLETION_RESERVE_MS,
+  HEARTBEAT_RESERVE_MS,
+  DRAIN_EVENT,
+  LIFECYCLE_EVENT,
+  LIFECYCLE_OUTCOME
 };