@fjall/components-infrastructure 39.0.1 → 39.0.2

package/dist/lib/patterns/aws/clickhouseDatabase.d.ts

@@ -41,10 +41,10 @@ export interface ClickHouseDatabaseProps {
      *  It is still not free: the type lives in the launch template and in
      *  user-data-baked server tuning, so it takes an instance refresh, and on
      *  this single-node cluster that refresh is a restart. The ASG terminates
-     *  the instance (TERMINATING hook detaches the volume), launches a
-     *  replacement, and the LAUNCHING hook re-attaches — ClickHouse is
-     *  unavailable across that window. Size the change into a quiet period;
-     *  do not plan a snapshot-restore migration for it. */
+     *  the instance and launches a replacement, and the data volume moves
+     *  across by the path `storageGb` describes — ClickHouse is unavailable
+     *  across that window. Size the change into a quiet period; do not plan a
+     *  snapshot-restore migration for it. */
     instanceType?: string;
     /**
      * EBS data-volume size in GiB. Default `CLICKHOUSE_EBS_VOLUME_SIZE_GB`
@@ -64,10 +64,22 @@ export interface ClickHouseDatabaseProps {
      * is embedded in that user data, so changing this prop versions the launch
      * template and triggers the ASG instance refresh whose boot grows the
      * filesystem — the increase applies in the same deploy, by the same
-     * mechanism an `instanceType` change rides. On this single-node cluster
-     * that relaunch is a restart (the TERMINATING hook detaches the volume,
-     * LAUNCHING re-attaches it): the data survives; the cluster is unavailable
-     * across the window. Size increases into a quiet period, exactly as with
+     * mechanism an `instanceType` change rides.
+     *
+     * On this single-node cluster that relaunch is a restart, and the data
+     * survives it because nothing detaches the volume. The TERMINATING hook
+     * drains the old instance and waits until every task on it is proven
+     * STOPPED, stopping any that outlast the drain and alarming when it cannot
+     * prove it; the instance's own shutdown then unmounts the filesystem and
+     * releases the volume. The LAUNCHING hook waits for that release, attaches
+     * the volume to the replacement, and ABANDONs the replacement if the
+     * volume is not released or attached in time, so the ASG launches another
+     * and the wait starts over. At boot the replacement formats the device
+     * only when it carries no filesystem signature, reads back, and holds the
+     * ext4 magic in neither its primary superblock nor its first backup; an
+     * ext4 filesystem is mounted as found, and anything else is refused rather
+     * than overwritten. The cluster is unavailable across the
+     * window, so size increases into a quiet period, exactly as with
      * `instanceType`.
      */
     storageGb?: number;

package/dist/lib/resources/aws/compute/asgInlineLifecycleHook.d.ts

@@ -40,13 +40,22 @@ export interface InlineAsgLifecycleHookProps {
  * events on the account+region default bus for every lifecycle hook regardless
  * of whether a notification target is configured. Each consumer attaches a
  * `Subscription` (from `lib/resources/aws/messaging/subscription.ts`) whose
- * event pattern discriminates by `AutoScalingGroupName` + `LifecycleHookName`,
+ * event pattern discriminates by `LifecycleHookName` alone — the name embeds
+ * the stack name, and matching on `AutoScalingGroupName` would Ref the ASG —
  * targeting the consumer's own SQS queue. The Lambda sees the EventBridge event
  * envelope and reads `detail.LifecycleActionToken` etc. from the unwrapped
  * detail.
  *
+ * The ASG depends on `owner`, the construct that consumes the hook's events.
+ * A hook whose rule, queue or Lambda does not exist yet drops its event and
+ * runs out its heartbeat, so the consumer must be created before the ASG
+ * launches, updated before an instance refresh replaces an instance, and
+ * deleted only after the ASG has terminated its instances. The consumer
+ * therefore must never Ref the ASG; the hook name and a wildcard ASG ARN are
+ * how the consumers avoid one.
+ *
  * Multiple consumers may call this helper against the same ASG — the existing
  * spec list is read, the new entry appended, and the merged array assigned
  * back. Synth is sequential per scope, so the merge is race-free.
  */
-export declare function attachInlineAsgLifecycleHook(_scope: Construct, _id: string, props: InlineAsgLifecycleHookProps): void;
+export declare function attachInlineAsgLifecycleHook(owner: Construct, _id: string, props: InlineAsgLifecycleHookProps): void;

package/dist/lib/resources/aws/compute/asgInlineLifecycleHook.js

@@ -26,16 +26,26 @@
  * events on the account+region default bus for every lifecycle hook regardless
  * of whether a notification target is configured. Each consumer attaches a
  * `Subscription` (from `lib/resources/aws/messaging/subscription.ts`) whose
- * event pattern discriminates by `AutoScalingGroupName` + `LifecycleHookName`,
+ * event pattern discriminates by `LifecycleHookName` alone — the name embeds
+ * the stack name, and matching on `AutoScalingGroupName` would Ref the ASG —
  * targeting the consumer's own SQS queue. The Lambda sees the EventBridge event
  * envelope and reads `detail.LifecycleActionToken` etc. from the unwrapped
  * detail.
  *
+ * The ASG depends on `owner`, the construct that consumes the hook's events.
+ * A hook whose rule, queue or Lambda does not exist yet drops its event and
+ * runs out its heartbeat, so the consumer must be created before the ASG
+ * launches, updated before an instance refresh replaces an instance, and
+ * deleted only after the ASG has terminated its instances. The consumer
+ * therefore must never Ref the ASG; the hook name and a wildcard ASG ARN are
+ * how the consumers avoid one.
+ *
  * Multiple consumers may call this helper against the same ASG — the existing
  * spec list is read, the new entry appended, and the merged array assigned
  * back. Synth is sequential per scope, so the merge is race-free.
  */
-export function attachInlineAsgLifecycleHook(_scope, _id, props) {
+export function attachInlineAsgLifecycleHook(owner, _id, props) {
+    props.autoScalingGroup.node.addDependency(owner);
     const cfnAsg = props.autoScalingGroup.node
         .defaultChild;
     const existing = readLifecycleHookSpecList(cfnAsg);

package/dist/lib/resources/aws/compute/ec2.d.ts

@@ -3,6 +3,7 @@ import { Construct } from "constructs";
 import { type Ec2SshAccessProps } from "./ec2SshAccess.js";
 import { Duration, Stack, type StackProps } from "aws-cdk-lib";
 import { type Role } from "aws-cdk-lib/aws-iam";
+import type { ITopic } from "aws-cdk-lib/aws-sns";
 import { AutoScalingGroup, Monitoring, type WarmPoolOptions } from "aws-cdk-lib/aws-autoscaling";
 import { type PersistentDataVolumeProps } from "./persistentDataVolume.js";
 export type Ec2InstancePersistentDataVolumeConfig = Omit<PersistentDataVolumeProps, "autoScalingGroup" | "stableOwnerId">;
@@ -114,13 +115,19 @@ export interface Ec2InstanceProps extends StackProps {
      * Bare-EC2 consumers (bastion, Fivetran) leave this unset.
      */
     ecsClusterArn?: string;
+    /**
+     * SNS topic for the graceful-termination handler's alarms: its queue's
+     * dead letters and, with `ecsClusterArn`, a terminating instance whose
+     * tasks were not proven STOPPED.
+     */
+    alertsTopic?: ITopic;
     /**
      * Pairs the ASG with a standalone EBS data volume that re-attaches across
      * instance refreshes. When set, requires `vpcSubnets.availabilityZones` to
      * be exactly one entry (matching `persistentDataVolume.availabilityZone`);
-     * the wrapper's volume is AZ-local and cannot follow a multi-AZ ASG.
-     * Forwards the wrapper's `ownerLogicalId` into the graceful-termination
-     * Lambda so the TERMINATING and LAUNCHING handlers locate the same volume.
+     * the wrapper's volume is AZ-local and cannot follow a multi-AZ ASG. It
+     * cannot be combined with `warmPool`: a single-attach volume cannot follow
+     * a pre-warmed instance.
      */
     persistentDataVolume?: Ec2InstancePersistentDataVolumeConfig;
     /**

package/dist/lib/resources/aws/compute/ec2.js

@@ -222,7 +222,7 @@ export class Ec2Instance extends Construct {
                 updatePolicy: resolvedUpdatePolicy
             }),
             // ECS task drain on terminate is handled by the EC2_INSTANCE_TERMINATING
-            // lifecycle hook + Ec2GracefulTerminationHandler (300s heartbeat), not by
+            // lifecycle hook + Ec2GracefulTerminationHandler, not by
             // managed termination protection. Leaving instances scale-in-protected
             // wedges CFN rollback because ASG cannot terminate the instance.
             newInstancesProtectedFromScaleIn: false,
@@ -289,15 +289,14 @@ export class Ec2Instance extends Construct {
      * ASGs (ClickHouse) without duplicate plumbing in the ECS layer.
      */
     addGracefulTerminationHandler(props) {
-        const dataVolumeOwnerLogicalId = this.persistentDataVolume?.ownerLogicalId;
         new Ec2GracefulTerminationHandler(this, `${props.serviceName}GracefulTermination`, {
             autoScalingGroup: this.autoScalingGroup,
             ...(props.ecsClusterArn !== undefined &&
                 props.ecsClusterArn !== "" && {
                 ecsClusterArn: props.ecsClusterArn
             }),
-            ...(dataVolumeOwnerLogicalId !== undefined && {
-                dataVolumeOwnerLogicalId
+            ...(props.alertsTopic !== undefined && {
+                alertsTopic: props.alertsTopic
             })
         });
     }
@@ -362,6 +361,11 @@ function validateEc2InstanceProps(props) {
         if (azs[0] !== props.persistentDataVolume.availabilityZone) {
             throw new Error(`Ec2Instance.persistentDataVolume.availabilityZone must match vpcSubnets.availabilityZones[0]; got '${props.persistentDataVolume.availabilityZone}' vs '${azs[0]}'`);
         }
+        if (props.warmPool !== undefined) {
+            throw new Error("Ec2Instance.persistentDataVolume cannot be combined with warmPool — " +
+                "the volume attaches to one instance at a time, so a pre-warmed " +
+                "instance can never receive it and its launch is abandoned on every attempt");
+        }
     }
     if (props.tags !== undefined) {
         for (const [key, value] of Object.entries(props.tags)) {

package/dist/lib/resources/aws/compute/ec2GracefulTerminationHandler.d.ts

@@ -1,3 +1,4 @@
+import type { ITopic } from "aws-cdk-lib/aws-sns";
 import { type AutoScalingGroup } from "aws-cdk-lib/aws-autoscaling";
 import { Construct } from "constructs";
 import { LambdaFunction } from "./lambda.js";
@@ -9,6 +10,12 @@ import { SQSQueue } from "../messaging/sqs.js";
  * source of truth so a rename does not silently break the test query.
  */
 export declare const EC2_GRACEFUL_TERMINATION_HANDLER_DESCRIPTION = "EC2 graceful termination handler";
+/**
+ * The Lambda logs this term when it cannot prove every task on a terminating
+ * instance STOPPED (`DRAIN_EVENT.UNPROVEN` in the Lambda source; a test pins
+ * the two together).
+ */
+export declare const DRAIN_UNPROVEN_EVENT = "drain_unproven";
 export interface Ec2GracefulTerminationHandlerProps {
     /** ASG to attach the EC2_INSTANCE_TERMINATING hook to. */
     autoScalingGroup: AutoScalingGroup;
@@ -20,11 +27,16 @@ export interface Ec2GracefulTerminationHandlerProps {
      */
     ecsClusterArn?: string;
     /**
-     * `fjall:OwnerLogicalId` of a paired `PersistentDataVolume`. When set, the
-     * Lambda detaches the tagged volume from the terminating instance before
-     * `CompleteLifecycleAction`. Empty string is normalised to `undefined`.
-     * Pass the `PersistentDataVolume.ownerLogicalId` so the TERMINATING and
-     * LAUNCHING handlers locate the same volume.
+     * SNS topic for the queue's dead-letter alarm and, when `ecsClusterArn` is
+     * set, the alarm that fires when a terminating instance's tasks are not
+     * proven STOPPED.
+     */
+    alertsTopic?: ITopic;
+    /**
+     * Ignored. The handler no longer detaches a paired `PersistentDataVolume`:
+     * the terminating instance's shutdown releases it.
+     *
+     * @deprecated Removed in 40.0.0; stop passing it.
      */
     dataVolumeOwnerLogicalId?: string;
 }

package/dist/lib/resources/aws/compute/ec2GracefulTerminationHandler.js

@@ -1,7 +1,7 @@
 import { readFileSync } from "node:fs";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
-import { Aws, Stack } from "aws-cdk-lib";
+import { Aws, Duration, Stack } from "aws-cdk-lib";
 import { Code, Runtime } from "aws-cdk-lib/aws-lambda";
 import { PolicyStatement, Effect } from "aws-cdk-lib/aws-iam";
 import { RetentionDays } from "aws-cdk-lib/aws-logs";
@@ -9,9 +9,10 @@ import { LifecycleTransition, DefaultResult } from "aws-cdk-lib/aws-autoscaling"
 import { Construct } from "constructs";
 import { attachInlineAsgLifecycleHook } from "./asgInlineLifecycleHook.js";
 import { LambdaFunction } from "./lambda.js";
-import { PERSISTENT_DATA_VOLUME_TAG_LIFECYCLE, PERSISTENT_DATA_VOLUME_TAG_LIFECYCLE_VALUE, PERSISTENT_DATA_VOLUME_TAG_STACK_ID } from "./persistentDataVolume.js";
 import { SQSQueue } from "../messaging/sqs.js";
 import { Subscription } from "../messaging/subscription.js";
+import { createLogPatternAlarms } from "../monitoring/logPatternAlarms.js";
+import { METRIC_NAMESPACE, stackScopedMetricNamespace } from "../monitoring/metricNamespaces.js";
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const LAMBDA_SOURCE_FILE = "ec2GracefulTerminationLambda.source.cjs";
 /**
@@ -21,11 +22,21 @@ const LAMBDA_SOURCE_FILE = "ec2GracefulTerminationLambda.source.cjs";
  * source of truth so a rename does not silently break the test query.
  */
 export const EC2_GRACEFUL_TERMINATION_HANDLER_DESCRIPTION = "EC2 graceful termination handler";
-const LAMBDA_TIMEOUT_SECONDS = 300;
-// Heartbeat must comfortably exceed the Lambda's drain (240s) + detach (60s)
-// budget so the hook does not time out mid-cleanup when PDV is wired.
-const HOOK_HEARTBEAT_SECONDS = 600;
-const QUEUE_VISIBILITY_TIMEOUT_SECONDS = 360;
+/**
+ * The Lambda logs this term when it cannot prove every task on a terminating
+ * instance STOPPED (`DRAIN_EVENT.UNPROVEN` in the Lambda source; a test pins
+ * the two together).
+ */
+export const DRAIN_UNPROVEN_EVENT = "drain_unproven";
+// Covers the Lambda's drain (240 s) and StopTask (150 s) budgets plus the
+// 90 s it keeps back for deregistration, cleanup and completion.
+const LAMBDA_TIMEOUT_SECONDS = 540;
+// SQS requires a visibility timeout of at least the function timeout.
+const QUEUE_VISIBILITY_TIMEOUT_SECONDS = 600;
+// Outlasts one full attempt and the redelivery of a timed-out one. The Lambda
+// reads it too: a proof read after the heartbeat could have released the
+// instance does not count.
+const HOOK_HEARTBEAT_SECONDS = 900;
 /**
  * ASG `EC2_INSTANCE_TERMINATING` hook + Lambda that drains the instance
  * cleanly. Drain ownership lives at the EC2 layer; ECS-specific drain plugs
@@ -40,29 +51,36 @@ export class Ec2GracefulTerminationHandler extends Construct {
         const sourcePath = path.resolve(__dirname, LAMBDA_SOURCE_FILE);
         const source = readFileSync(sourcePath, "utf-8");
         const ecsClusterArn = resolveOptionalString(props.ecsClusterArn);
-        const dataVolumeOwnerLogicalId = resolveOptionalString(props.dataVolumeOwnerLogicalId);
         this.queue = new SQSQueue(this, `${id}Queue`, {
             visibilityTimeout: QUEUE_VISIBILITY_TIMEOUT_SECONDS,
             deadLetterQueue: { enabled: true, maxReceiveCount: 5 },
             // Transient instance-drain signals — no durable state. Pinned DESTROY
             // (now also the SQSQueue wrapper default) so a replacing deploy of the
             // parent Ec2Instance reclaims this queue + DLQ instead of orphaning them.
-            removalPolicy: "DESTROY"
+            removalPolicy: "DESTROY",
+            ...(props.alertsTopic !== undefined && { alertsTopic: props.alertsTopic })
         });
         const stack = Stack.of(this);
         const ecsPolicies = ecsClusterArn !== undefined
             ? [
+                // Both act on the cluster resource and carry no `ecs:cluster`
+                // condition key (AWS service authorization reference), so a
+                // condition on that key would deny them.
                 new PolicyStatement({
                     effect: Effect.ALLOW,
-                    actions: ["ecs:ListContainerInstances"],
+                    actions: [
+                        "ecs:ListContainerInstances",
+                        "ecs:DeregisterContainerInstance"
+                    ],
                     resources: [ecsClusterArn]
                 }),
                 new PolicyStatement({
                     effect: Effect.ALLOW,
                     actions: [
-                        "ecs:DescribeContainerInstances",
                         "ecs:UpdateContainerInstancesState",
-                        "ecs:DeregisterContainerInstance"
+                        "ecs:ListTasks",
+                        "ecs:DescribeTasks",
+                        "ecs:StopTask"
                     ],
                     resources: ["*"],
                     conditions: {
@@ -116,33 +134,6 @@ export class Ec2GracefulTerminationHandler extends Construct {
             actions: ["autoscaling:CompleteLifecycleAction"],
             resources: [asgArnWildcard]
         });
-        const volumeArnPattern = `arn:${stack.partition}:ec2:${stack.region}:${stack.account}:volume/*`;
-        const instanceArnPattern = `arn:${stack.partition}:ec2:${stack.region}:${stack.account}:instance/*`;
-        const dataVolumePolicies = dataVolumeOwnerLogicalId !== undefined
-            ? [
-                new PolicyStatement({
-                    effect: Effect.ALLOW,
-                    actions: ["ec2:DescribeVolumes"],
-                    resources: ["*"]
-                }),
-                new PolicyStatement({
-                    effect: Effect.ALLOW,
-                    actions: ["ec2:DetachVolume"],
-                    resources: [volumeArnPattern],
-                    conditions: {
-                        StringEquals: {
-                            [`aws:ResourceTag/${PERSISTENT_DATA_VOLUME_TAG_LIFECYCLE}`]: PERSISTENT_DATA_VOLUME_TAG_LIFECYCLE_VALUE,
-                            [`aws:ResourceTag/${PERSISTENT_DATA_VOLUME_TAG_STACK_ID}`]: Aws.STACK_ID
-                        }
-                    }
-                }),
-                new PolicyStatement({
-                    effect: Effect.ALLOW,
-                    actions: ["ec2:DetachVolume"],
-                    resources: [instanceArnPattern]
-                })
-            ]
-            : [];
         this.lambda = new LambdaFunction(this, `${id}Fn`, {
             runtime: Runtime.NODEJS_24_X,
             handler: "index.handler",
@@ -153,10 +144,9 @@ export class Ec2GracefulTerminationHandler extends Construct {
             memorySize: 256,
             logGroupRetention: RetentionDays.ONE_MONTH,
             environment: {
-                ...(ecsClusterArn !== undefined && { ECS_CLUSTER_ARN: ecsClusterArn }),
-                ...(dataVolumeOwnerLogicalId !== undefined && {
-                    DATA_VOLUME_OWNER_LOGICAL_ID: dataVolumeOwnerLogicalId,
-                    DATA_VOLUME_STACK_ID: Aws.STACK_ID
+                ...(ecsClusterArn !== undefined && {
+                    ECS_CLUSTER_ARN: ecsClusterArn,
+                    LIFECYCLE_HEARTBEAT_SECONDS: String(HOOK_HEARTBEAT_SECONDS)
                 })
             },
             inlinePolicy: [
@@ -164,11 +154,30 @@ export class Ec2GracefulTerminationHandler extends Construct {
                 ec2Policies,
                 elbReadPolicy,
                 elbDeregisterPolicy,
-                asgPolicy,
-                ...dataVolumePolicies
+                asgPolicy
             ]
         });
-        this.lambda.addSqsEventSource(this.queue.getQueue());
+        if (ecsClusterArn !== undefined && props.alertsTopic !== undefined) {
+            createLogPatternAlarms({
+                scope: this,
+                logGroup: this.lambda.functionLogGroup,
+                alarmTopic: props.alertsTopic,
+                metricNamespace: stackScopedMetricNamespace(METRIC_NAMESPACE.EC2),
+                specs: [
+                    {
+                        idStem: `${id}DrainUnproven`,
+                        metricName: `${id}DrainUnprovenCount`,
+                        description: `${id}: a terminating instance's ECS tasks were not proven STOPPED ` +
+                            "before it was released — check the handler's log for the unstopped task ARNs",
+                        anyTerms: [DRAIN_UNPROVEN_EVENT],
+                        period: Duration.minutes(5)
+                    }
+                ]
+            });
+        }
+        // One lifecycle action per invocation: a batch is processed in turn, so
+        // several instances terminating together would share one timeout.
+        this.lambda.addSqsEventSource(this.queue.getQueue(), { batchSize: 1 });
         // Hook name embeds Aws.STACK_NAME (pseudo-parameter Ref, NOT a resource
         // Ref — no CFN dep) so LifecycleHookName alone discriminates across
         // stacks in the EB pattern. Sibling LAUNCHING hook uses "-launching".