@fjall/components-infrastructure 9.0.0 → 10.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/patterns/aws/compute.js +13 -1
- package/dist/lib/patterns/aws/computeEcs.d.ts +80 -1
- package/dist/lib/patterns/aws/computeEcs.js +450 -70
- package/dist/lib/patterns/aws/computeEcsTypes.d.ts +113 -6
- package/dist/lib/patterns/aws/computeEcsTypes.js +19 -0
- package/dist/lib/patterns/aws/computePropApplicability.js +1 -0
- package/dist/lib/patterns/aws/database.d.ts +12 -1
- package/dist/lib/patterns/aws/database.js +16 -0
- package/dist/lib/patterns/aws/devSubstrate.js +64 -8
- package/dist/lib/patterns/aws/interfaces/database.d.ts +26 -0
- package/dist/lib/patterns/aws/storage.d.ts +7 -0
- package/dist/lib/patterns/aws/storage.js +14 -1
- package/dist/lib/resources/aws/compute/applicationLoadBalancer.js +6 -0
- package/dist/lib/resources/aws/compute/ecs.d.ts +12 -0
- package/dist/lib/resources/aws/compute/ecs.js +32 -2
- package/dist/lib/resources/aws/compute/ecsImages.js +4 -0
- package/dist/lib/resources/aws/compute/ecsServiceFactory.d.ts +29 -3
- package/dist/lib/resources/aws/compute/ecsServiceFactory.js +32 -0
- package/dist/lib/resources/aws/compute/ecsTaskDefinition.js +12 -0
- package/dist/lib/resources/aws/compute/ecsTypes.d.ts +42 -2
- package/dist/lib/resources/aws/compute/ecsValidation.d.ts +6 -0
- package/dist/lib/resources/aws/compute/ecsValidation.js +13 -0
- package/dist/lib/resources/aws/database/rdsAurora.d.ts +7 -1
- package/dist/lib/resources/aws/database/rdsAurora.js +1 -1
- package/dist/lib/resources/aws/messaging/eventTargets.d.ts +4 -2
- package/dist/lib/resources/aws/messaging/eventTargets.js +55 -2
- package/dist/lib/resources/aws/monitoring/alarmDefaults.d.ts +1 -0
- package/dist/lib/resources/aws/monitoring/alarmDefaults.js +1 -1
- package/dist/lib/resources/aws/monitoring/ecsTaskStopWatchdog.d.ts +59 -0
- package/dist/lib/resources/aws/monitoring/ecsTaskStopWatchdog.js +115 -0
- package/dist/lib/resources/aws/monitoring/index.d.ts +1 -0
- package/dist/lib/resources/aws/monitoring/index.js +1 -0
- package/dist/lib/resources/aws/monitoring/metricNamespaces.d.ts +1 -0
- package/dist/lib/resources/aws/monitoring/metricNamespaces.js +1 -0
- package/dist/lib/utils/manifestWriter.d.ts +16 -2
- package/dist/lib/utils/manifestWriter.js +22 -0
- package/package.json +4 -3
|
@@ -11,10 +11,28 @@ import type { EcsConstructContext } from "./ecsContext.js";
|
|
|
11
11
|
* `DeploymentCircuitBreaker` shape. `undefined` resolves to the safe default
|
|
12
12
|
* `{ enable: true, rollback: true }`; `false` resolves to `undefined` (CDK
|
|
13
13
|
* omits the breaker block from CFN output entirely).
|
|
14
|
+
*
|
|
15
|
+
* Covers only the halves the L2 type can express (`enable`/`rollback`);
|
|
16
|
+
* `resetOnHealthyTask` and `threshold` land via
|
|
17
|
+
* {@link applyCircuitBreakerHardening} on the synthesised L1.
|
|
14
18
|
*/
|
|
15
|
-
export declare function resolveCircuitBreaker(config:
|
|
16
|
-
|
|
17
|
-
|
|
19
|
+
export declare function resolveCircuitBreaker(config: EcsServiceProps["circuitBreaker"] | undefined): DeploymentCircuitBreaker | undefined;
|
|
20
|
+
/**
|
|
21
|
+
* Applies the breaker properties CDK's L2 `DeploymentCircuitBreaker` type
|
|
22
|
+
* cannot express, via property overrides on the synthesised `CfnService`:
|
|
23
|
+
*
|
|
24
|
+
* - `ResetOnHealthyTask` defaults to `false` — deliberately stricter than
|
|
25
|
+
* AWS's `true` default, where any one healthy task resets the failure
|
|
26
|
+
* counter and a slow crash loop that occasionally boots can hold a
|
|
27
|
+
* deployment `IN_PROGRESS` indefinitely.
|
|
28
|
+
* - `ThresholdConfiguration { Type: COUNT }` when `threshold` is set —
|
|
29
|
+
* an absolute failed-task count is the comprehensible shape for the
|
|
30
|
+
* low-desired-count workers this hardening exists for.
|
|
31
|
+
*
|
|
32
|
+
* No-op when the breaker is disabled (`config === false`) — CFN rejects
|
|
33
|
+
* breaker sub-properties without an enclosing breaker block.
|
|
34
|
+
*/
|
|
35
|
+
export declare function applyCircuitBreakerHardening(service: FargateService | Ec2Service, config: EcsServiceProps["circuitBreaker"] | undefined): void;
|
|
18
36
|
/** Mutable state for ASG capacity provider deduplication, keyed by SLOT. */
|
|
19
37
|
export interface AsgCapacityState {
|
|
20
38
|
providers: Map<string, AsgCapacityProvider>;
|
|
@@ -27,6 +45,14 @@ export interface AsgCapacityState {
|
|
|
27
45
|
asgOrigins: Map<string, Ec2AsgOrigin>;
|
|
28
46
|
autoScalingGroup?: AutoScalingGroup;
|
|
29
47
|
asgSecurityGroup?: ISecurityGroup;
|
|
48
|
+
/**
|
|
49
|
+
* Every capacity slot's instance security group (deduped — a cluster-level
|
|
50
|
+
* `securityGroup` is shared across slots). Consumers that must reach tasks
|
|
51
|
+
* regardless of which slot's instances they land on (EventBridge scheduled
|
|
52
|
+
* tasks place unconstrained across capacity providers) use this, not the
|
|
53
|
+
* first-wins singular field above.
|
|
54
|
+
*/
|
|
55
|
+
asgSecurityGroups: ISecurityGroup[];
|
|
30
56
|
}
|
|
31
57
|
export { getEc2ConfigKey, validateSharedEc2CapacityConfig, type Ec2AsgOrigin, type SharedEc2CapacityServiceInput } from "./ecsCapacityConfig.js";
|
|
32
58
|
/**
|
|
@@ -22,6 +22,10 @@ import { isServiceFargate, isServiceEc2 } from "./ecsTaskDefinition.js";
|
|
|
22
22
|
* `DeploymentCircuitBreaker` shape. `undefined` resolves to the safe default
|
|
23
23
|
* `{ enable: true, rollback: true }`; `false` resolves to `undefined` (CDK
|
|
24
24
|
* omits the breaker block from CFN output entirely).
|
|
25
|
+
*
|
|
26
|
+
* Covers only the halves the L2 type can express (`enable`/`rollback`);
|
|
27
|
+
* `resetOnHealthyTask` and `threshold` land via
|
|
28
|
+
* {@link applyCircuitBreakerHardening} on the synthesised L1.
|
|
25
29
|
*/
|
|
26
30
|
export function resolveCircuitBreaker(config) {
|
|
27
31
|
if (config === false)
|
|
@@ -29,6 +33,30 @@ export function resolveCircuitBreaker(config) {
|
|
|
29
33
|
const rollback = config?.rollback ?? true;
|
|
30
34
|
return { enable: true, rollback };
|
|
31
35
|
}
|
|
36
|
+
/**
|
|
37
|
+
* Applies the breaker properties CDK's L2 `DeploymentCircuitBreaker` type
|
|
38
|
+
* cannot express, via property overrides on the synthesised `CfnService`:
|
|
39
|
+
*
|
|
40
|
+
* - `ResetOnHealthyTask` defaults to `false` — deliberately stricter than
|
|
41
|
+
* AWS's `true` default, where any one healthy task resets the failure
|
|
42
|
+
* counter and a slow crash loop that occasionally boots can hold a
|
|
43
|
+
* deployment `IN_PROGRESS` indefinitely.
|
|
44
|
+
* - `ThresholdConfiguration { Type: COUNT }` when `threshold` is set —
|
|
45
|
+
* an absolute failed-task count is the comprehensible shape for the
|
|
46
|
+
* low-desired-count workers this hardening exists for.
|
|
47
|
+
*
|
|
48
|
+
* No-op when the breaker is disabled (`config === false`) — CFN rejects
|
|
49
|
+
* breaker sub-properties without an enclosing breaker block.
|
|
50
|
+
*/
|
|
51
|
+
export function applyCircuitBreakerHardening(service, config) {
|
|
52
|
+
if (config === false)
|
|
53
|
+
return;
|
|
54
|
+
const cfnService = service.node.defaultChild;
|
|
55
|
+
cfnService.addPropertyOverride("DeploymentConfiguration.DeploymentCircuitBreaker.ResetOnHealthyTask", config?.resetOnHealthyTask ?? false);
|
|
56
|
+
if (config?.threshold !== undefined) {
|
|
57
|
+
cfnService.addPropertyOverride("DeploymentConfiguration.DeploymentCircuitBreaker.ThresholdConfiguration", { Type: "COUNT", Value: config.threshold });
|
|
58
|
+
}
|
|
59
|
+
}
|
|
32
60
|
export { getEc2ConfigKey, validateSharedEc2CapacityConfig } from "./ecsCapacityConfig.js";
|
|
33
61
|
/**
|
|
34
62
|
* Gets or creates an ASG capacity provider for an EC2-backed service.
|
|
@@ -196,6 +224,9 @@ export function getOrCreateAsgCapacityProvider(ctx, serviceProps, state) {
|
|
|
196
224
|
if (!state.asgSecurityGroup) {
|
|
197
225
|
state.asgSecurityGroup = asgSecurityGroup;
|
|
198
226
|
}
|
|
227
|
+
if (!state.asgSecurityGroups.includes(asgSecurityGroup)) {
|
|
228
|
+
state.asgSecurityGroups.push(asgSecurityGroup);
|
|
229
|
+
}
|
|
199
230
|
return provider;
|
|
200
231
|
}
|
|
201
232
|
/**
|
|
@@ -268,6 +299,7 @@ export function createService(ctx, serviceName, serviceProps, taskDefinition, as
|
|
|
268
299
|
})
|
|
269
300
|
});
|
|
270
301
|
}
|
|
302
|
+
applyCircuitBreakerHardening(service, serviceProps.circuitBreaker);
|
|
271
303
|
if (serviceProps.cloudMapService !== undefined) {
|
|
272
304
|
const isSrv = serviceProps.cloudMapDnsRecordType === "SRV";
|
|
273
305
|
const isHostOrBridge = taskDefinition.networkMode === NetworkMode.HOST ||
|
|
@@ -3,6 +3,7 @@ import { Duration } from "aws-cdk-lib";
|
|
|
3
3
|
import { Secret as EcsSecret } from "aws-cdk-lib/aws-ecs";
|
|
4
4
|
import { StringParameter } from "aws-cdk-lib/aws-ssm";
|
|
5
5
|
import { buildParameterPath } from "@fjall/util";
|
|
6
|
+
import { DEFAULT_ALB_IDLE_TIMEOUT_SECONDS, FJALL_ALB_IDLE_TIMEOUT_ENV_VAR } from "@fjall/util/httpKeepAlive";
|
|
6
7
|
import { resolveOrgId } from "../../../utils/cdkContext.js";
|
|
7
8
|
import { validateSsmPathComponent, validateSecretName } from "./ecsValidation.js";
|
|
8
9
|
import { DEFAULT_LOG_RETENTION, DEFAULT_FARGATE_CPU, DEFAULT_FARGATE_MEMORY_MIB, resolveEc2ContainerMemoryMiB } from "./ecsConstants.js";
|
|
@@ -194,6 +195,14 @@ export function addContainersToTask(ctx, serviceName, serviceProps, taskDefiniti
|
|
|
194
195
|
...remoteEnv,
|
|
195
196
|
...(containerConfig.port
|
|
196
197
|
? { PORT: String(containerConfig.port) }
|
|
198
|
+
: {}),
|
|
199
|
+
// The ALB half of the keep-alive contract (@fjall/util/httpKeepAlive):
|
|
200
|
+
// only the first port-bearing container is ALB-registered, and the
|
|
201
|
+
// value must match the idleTimeout pinned in applicationLoadBalancer.ts.
|
|
202
|
+
...(isFirstWithPort && !ctx.loadBalancerDisabled
|
|
203
|
+
? {
|
|
204
|
+
[FJALL_ALB_IDLE_TIMEOUT_ENV_VAR]: String(DEFAULT_ALB_IDLE_TIMEOUT_SECONDS)
|
|
205
|
+
}
|
|
197
206
|
: {})
|
|
198
207
|
},
|
|
199
208
|
secrets,
|
|
@@ -218,6 +227,9 @@ export function addContainersToTask(ctx, serviceName, serviceProps, taskDefiniti
|
|
|
218
227
|
stopTimeout: containerConfig.stopTimeout !== undefined
|
|
219
228
|
? Duration.seconds(containerConfig.stopTimeout)
|
|
220
229
|
: undefined,
|
|
230
|
+
startTimeout: containerConfig.startTimeout !== undefined
|
|
231
|
+
? Duration.seconds(containerConfig.startTimeout)
|
|
232
|
+
: undefined,
|
|
221
233
|
...(isServiceEc2(serviceProps) && {
|
|
222
234
|
memoryLimitMiB: resolveEc2ContainerMemoryMiB(serviceProps.ec2Config)
|
|
223
235
|
})
|
|
@@ -334,6 +334,15 @@ export interface EcsClusterContainerConfig {
|
|
|
334
334
|
* - Repository: CDK ECR Repository construct
|
|
335
335
|
*/
|
|
336
336
|
image?: string | Repository;
|
|
337
|
+
/**
|
|
338
|
+
* When true, a string `image` is always rendered verbatim via
|
|
339
|
+
* `ContainerImage.fromRegistry` — never interpreted as an ECR repository
|
|
340
|
+
* name with the deploy-managed `<Service>ImageTag` parameter appended.
|
|
341
|
+
* Set by the patterns layer for synthetic containers whose image is
|
|
342
|
+
* release-pinned (the schema gate); the registry-URL heuristic in
|
|
343
|
+
* `getContainerImage` cannot recognise arbitrary custom registry hosts.
|
|
344
|
+
*/
|
|
345
|
+
verbatimImage?: boolean;
|
|
337
346
|
/**
|
|
338
347
|
* Port the container listens on.
|
|
339
348
|
* The first container with a port becomes the **primary container**
|
|
@@ -366,7 +375,10 @@ export interface EcsClusterContainerConfig {
|
|
|
366
375
|
essential?: boolean;
|
|
367
376
|
/**
|
|
368
377
|
* Health check configuration.
|
|
369
|
-
* Default:
|
|
378
|
+
* Default: none — the construct emits no container health check unless one
|
|
379
|
+
* is declared here. Without one (and without an ALB target group), ECS
|
|
380
|
+
* treats the container as healthy from the moment it is RUNNING, so
|
|
381
|
+
* post-start exits are invisible to the deployment circuit breaker.
|
|
370
382
|
*/
|
|
371
383
|
healthCheck?: {
|
|
372
384
|
command: string[];
|
|
@@ -401,6 +413,13 @@ export interface EcsClusterContainerConfig {
|
|
|
401
413
|
* SIGTERM before sending SIGKILL. Range 1–120. Default: ECS default (30s).
|
|
402
414
|
*/
|
|
403
415
|
stopTimeout?: number;
|
|
416
|
+
/**
|
|
417
|
+
* Time (seconds) dependents of THIS container wait for it to reach their
|
|
418
|
+
* declared `dependsOn` condition before giving up (which stops the task
|
|
419
|
+
* pre-RUNNING). Declared on the depended-on container, per ECS semantics.
|
|
420
|
+
* Default: none on Fargate; the agent's 3-minute default on EC2.
|
|
421
|
+
*/
|
|
422
|
+
startTimeout?: number;
|
|
404
423
|
}
|
|
405
424
|
/**
|
|
406
425
|
* Cluster-level configuration.
|
|
@@ -609,14 +628,25 @@ export interface EcsServiceProps {
|
|
|
609
628
|
logMetricNamespace?: string;
|
|
610
629
|
/**
|
|
611
630
|
* Deployment circuit breaker policy.
|
|
612
|
-
* - undefined (default): `{ enable: true, rollback: true }`
|
|
631
|
+
* - undefined (default): `{ enable: true, rollback: true }` with
|
|
632
|
+
* `resetOnHealthyTask: false`
|
|
613
633
|
* - `false`: disabled entirely (no breaker)
|
|
614
634
|
* - `{ rollback: boolean }`: override rollback behaviour
|
|
635
|
+
* - `resetOnHealthyTask` (default `false` — deliberately stricter than
|
|
636
|
+
* AWS's `true` default): when `true`, any one healthy task resets the
|
|
637
|
+
* breaker's failure counter, so a slow crash loop that occasionally
|
|
638
|
+
* boots can hold a deployment `IN_PROGRESS` indefinitely.
|
|
639
|
+
* - `threshold`: absolute failed-task count before the breaker trips
|
|
640
|
+
* (CFN `ThresholdConfiguration { Type: COUNT }`). Omitted → ECS computes
|
|
641
|
+
* its default from the desired count. Must be a positive integer
|
|
642
|
+
* (enforced by `validateEcsClusterProps`).
|
|
615
643
|
*
|
|
616
644
|
* @see https://docs.aws.amazon.com/AmazonECS/latest/developerguide/deployment-circuit-breaker.html
|
|
617
645
|
*/
|
|
618
646
|
circuitBreaker?: false | {
|
|
619
647
|
rollback?: boolean;
|
|
648
|
+
resetOnHealthyTask?: boolean;
|
|
649
|
+
threshold?: number;
|
|
620
650
|
};
|
|
621
651
|
/**
|
|
622
652
|
* Rolling-deploy capacity bounds. Overrides the default
|
|
@@ -698,6 +728,16 @@ export interface EcsClusterProps {
|
|
|
698
728
|
alertsTopic?: ITopic;
|
|
699
729
|
/** Application ID for alarm tagging (used by webhook to map alarms to applications). */
|
|
700
730
|
applicationId?: string;
|
|
731
|
+
/**
|
|
732
|
+
* ECS task-stop watchdog (default on): a per-cluster EventBridge rule
|
|
733
|
+
* captures abnormal task stops into a 30-day forensic log group
|
|
734
|
+
* `/fjall/<cluster>/task-stops`, derives
|
|
735
|
+
* `Fjall/ECS StoppedTaskCount{ClusterName,ServiceName}` via a metric
|
|
736
|
+
* filter, and (when `alertsTopic` is set) raises a notification-only
|
|
737
|
+
* churn alarm per service. `false` disables the whole watchdog —
|
|
738
|
+
* auditable opt-out for cost-sensitive clusters.
|
|
739
|
+
*/
|
|
740
|
+
taskStopWatchdog?: false;
|
|
701
741
|
}
|
|
702
742
|
/**
|
|
703
743
|
* Data tracked for each service in the cluster.
|
|
@@ -19,6 +19,12 @@ import type { DomainConfig, EcsCapacityProvider, EcsClusterProps } from "./ecsTy
|
|
|
19
19
|
* `new EcsCluster(...)` consumer cannot pass it — there is no resources-layer
|
|
20
20
|
* code path to validate.
|
|
21
21
|
*
|
|
22
|
+
* Same applies to `service.serviceType`: the `"web" | "worker"` intent marker
|
|
23
|
+
* is a patterns-layer field on `EcsServiceConfig` (its honesty check — worker
|
|
24
|
+
* + port/portMappings/routing → throw — runs in `validateEcsProps`). It is not
|
|
25
|
+
* a field on `EcsServiceProps`, so a direct `new EcsCluster(...)` consumer
|
|
26
|
+
* cannot pass it and there is no resources-layer code path to validate.
|
|
27
|
+
*
|
|
22
28
|
* @param props - The cluster props to validate
|
|
23
29
|
* @throws Error if validation fails
|
|
24
30
|
*/
|
|
@@ -24,6 +24,12 @@ import { validateSharedEc2CapacityConfig } from "./ecsCapacityConfig.js";
|
|
|
24
24
|
* `new EcsCluster(...)` consumer cannot pass it — there is no resources-layer
|
|
25
25
|
* code path to validate.
|
|
26
26
|
*
|
|
27
|
+
* Same applies to `service.serviceType`: the `"web" | "worker"` intent marker
|
|
28
|
+
* is a patterns-layer field on `EcsServiceConfig` (its honesty check — worker
|
|
29
|
+
* + port/portMappings/routing → throw — runs in `validateEcsProps`). It is not
|
|
30
|
+
* a field on `EcsServiceProps`, so a direct `new EcsCluster(...)` consumer
|
|
31
|
+
* cannot pass it and there is no resources-layer code path to validate.
|
|
32
|
+
*
|
|
27
33
|
* @param props - The cluster props to validate
|
|
28
34
|
* @throws Error if validation fails
|
|
29
35
|
*/
|
|
@@ -124,6 +130,13 @@ export function validateEcsClusterProps(props) {
|
|
|
124
130
|
"Provide ec2Config on the service.");
|
|
125
131
|
}
|
|
126
132
|
validateEc2ServiceSizing(service);
|
|
133
|
+
if (typeof service.circuitBreaker === "object") {
|
|
134
|
+
const threshold = service.circuitBreaker.threshold;
|
|
135
|
+
if (threshold !== undefined &&
|
|
136
|
+
(!Number.isInteger(threshold) || threshold < 1)) {
|
|
137
|
+
throw new Error(`Service '${service.name}': circuitBreaker.threshold must be a positive integer (got ${threshold}).`);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
127
140
|
if (service.deployment !== undefined) {
|
|
128
141
|
const min = service.deployment.minHealthyPercent;
|
|
129
142
|
const max = service.deployment.maxHealthyPercent;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Duration } from "aws-cdk-lib";
|
|
1
|
+
import { Duration, RemovalPolicy } from "aws-cdk-lib";
|
|
2
2
|
import { Connections, type IConnectable, type IVpc } from "aws-cdk-lib/aws-ec2";
|
|
3
3
|
import { DatabaseCluster, type CfnDBCluster, type IClusterEngine } from "aws-cdk-lib/aws-rds";
|
|
4
4
|
import { Construct } from "constructs";
|
|
@@ -49,6 +49,12 @@ interface RdsProps {
|
|
|
49
49
|
isGlobalSecondary?: boolean;
|
|
50
50
|
secretReplicaRegions?: string[];
|
|
51
51
|
deletionProtection?: boolean;
|
|
52
|
+
/**
|
|
53
|
+
* Defaults to SNAPSHOT (final snapshot on delete). DESTROY is reserved for
|
|
54
|
+
* disposable-data clusters (dev substrates) — delete takes the data with it,
|
|
55
|
+
* no snapshot, no recovery.
|
|
56
|
+
*/
|
|
57
|
+
removalPolicy?: RemovalPolicy;
|
|
52
58
|
/** ARN or identifier of DB cluster snapshot to restore from */
|
|
53
59
|
snapshotIdentifier?: string;
|
|
54
60
|
/**
|
|
@@ -180,7 +180,7 @@ export class RdsAurora extends Construct {
|
|
|
180
180
|
preferredMaintenanceWindow: props.preferredMaintenanceWindow ??
|
|
181
181
|
RDS_DEFAULTS.PREFERRED_MAINTENANCE_WINDOW,
|
|
182
182
|
port: this.port,
|
|
183
|
-
removalPolicy: RemovalPolicy.SNAPSHOT,
|
|
183
|
+
removalPolicy: props.removalPolicy ?? RemovalPolicy.SNAPSHOT,
|
|
184
184
|
deletionProtection: props.deletionProtection ?? true,
|
|
185
185
|
iamAuthentication: true,
|
|
186
186
|
...(props.serverlessV2MinCapacity !== undefined && {
|
|
@@ -4,6 +4,7 @@ import type { IFunction } from "aws-cdk-lib/aws-lambda";
|
|
|
4
4
|
import type { ICluster, TaskDefinition } from "aws-cdk-lib/aws-ecs";
|
|
5
5
|
import { SQSQueue } from "./sqs.js";
|
|
6
6
|
import { EventBridgeBus } from "./eventbridge.js";
|
|
7
|
+
import { LogGroup } from "../logging/logGroup.js";
|
|
7
8
|
import { CodeBuildProject } from "../utilities/codeBuild.js";
|
|
8
9
|
import type { EventBridgeRetryPolicy } from "./eventBridgeRule.js";
|
|
9
10
|
interface LambdaComputeShape {
|
|
@@ -33,10 +34,11 @@ export type ScheduleTargetInput = SQSQueue | LambdaComputeShape | EcsScheduleTar
|
|
|
33
34
|
/**
|
|
34
35
|
* Public input type for `Subscription.target`. Same dispatch convention as
|
|
35
36
|
* `ScheduleTargetInput`. Subscriptions accept `CodeBuildProject` (used by
|
|
36
|
-
* `EcrDefaultImage`)
|
|
37
|
+
* `EcrDefaultImage`) and `LogGroup` (forensic event capture, used by the ECS
|
|
38
|
+
* task-stop watchdog) but not `EventBridgeBus` — bus-to-bus replication is
|
|
37
39
|
* not a subscription shape.
|
|
38
40
|
*/
|
|
39
|
-
export type SubscriptionTargetInput = SQSQueue | LambdaComputeShape | EcsScheduleTarget | CodeBuildProject;
|
|
41
|
+
export type SubscriptionTargetInput = SQSQueue | LambdaComputeShape | EcsScheduleTarget | CodeBuildProject | LogGroup;
|
|
40
42
|
/**
|
|
41
43
|
* Internal resolve input — the union of every shape `resolveTarget(...)` can
|
|
42
44
|
* dispatch over. `Schedule` and `Subscription` narrow this at the public
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import { EventField, RuleTargetInput } from "aws-cdk-lib/aws-events";
|
|
2
|
-
import {
|
|
2
|
+
import { ArnFormat, Stack } from "aws-cdk-lib";
|
|
3
|
+
import { Effect, PolicyStatement, ServicePrincipal } from "aws-cdk-lib/aws-iam";
|
|
4
|
+
import { SqsQueue as SqsTarget, LambdaFunction as LambdaTarget, EventBus as EventBusTarget, CodeBuildProject as CodeBuildTarget, EcsTask as EcsTaskTarget, addToDeadLetterQueueResourcePolicy, bindBaseTargetConfig } from "aws-cdk-lib/aws-events-targets";
|
|
3
5
|
import { SQSQueue } from "./sqs.js";
|
|
4
6
|
import { EventBridgeBus } from "./eventbridge.js";
|
|
7
|
+
import { LogGroup } from "../logging/logGroup.js";
|
|
5
8
|
import { CodeBuildProject } from "../utilities/codeBuild.js";
|
|
6
9
|
function isEcsScheduleTarget(target) {
|
|
7
10
|
return (typeof target === "object" &&
|
|
@@ -63,13 +66,16 @@ export function resolveTarget(target, options) {
|
|
|
63
66
|
if (target instanceof CodeBuildProject) {
|
|
64
67
|
return resolveCodeBuildTarget(target, options.messageGroupId, baseProps, targetInput);
|
|
65
68
|
}
|
|
69
|
+
if (target instanceof LogGroup) {
|
|
70
|
+
return resolveLogGroupTarget(target, options.messageGroupId, baseProps, targetInput);
|
|
71
|
+
}
|
|
66
72
|
if (isEcsScheduleTarget(target)) {
|
|
67
73
|
return resolveEcsTarget(target, options.messageGroupId, baseProps, targetInput);
|
|
68
74
|
}
|
|
69
75
|
if (isLambdaCompute(target)) {
|
|
70
76
|
return resolveLambdaTarget(target, options.messageGroupId, baseProps, targetInput);
|
|
71
77
|
}
|
|
72
|
-
throw new Error(`Unsupported event target: expected SQSQueue/QueueMessaging, LambdaCompute, EventBridgeBus/EventBusMessaging, CodeBuildProject, or { ecs: EcsCompute, serviceName }; received ${describeTarget(target)}`);
|
|
78
|
+
throw new Error(`Unsupported event target: expected SQSQueue/QueueMessaging, LambdaCompute, EventBridgeBus/EventBusMessaging, CodeBuildProject, LogGroup, or { ecs: EcsCompute, serviceName }; received ${describeTarget(target)}`);
|
|
73
79
|
}
|
|
74
80
|
function resolveQueueTarget(target, messageGroupId, baseProps, targetInput) {
|
|
75
81
|
const queue = target.getQueue();
|
|
@@ -135,6 +141,53 @@ function resolveCodeBuildTarget(target, messageGroupId, baseProps, targetInput)
|
|
|
135
141
|
...(targetInput !== undefined && { event: targetInput })
|
|
136
142
|
});
|
|
137
143
|
}
|
|
144
|
+
/**
|
|
145
|
+
* CloudWatch Logs rule target. Hand-rolled rather than CDK's
|
|
146
|
+
* `CloudWatchLogGroup` adapter: that adapter provisions the log-group resource
|
|
147
|
+
* policy through a custom-resource Lambda (`LogGroupResourcePolicy`), adding a
|
|
148
|
+
* Lambda to every consuming stack plus one account-level Logs resource policy
|
|
149
|
+
* per rule — and account-level policies are hard-capped at 10 per region.
|
|
150
|
+
* `addToResourcePolicy` synthesises a native `AWS::Logs::ResourcePolicy`
|
|
151
|
+
* instead: no Lambda, one policy per log group, deduplicated by CDK.
|
|
152
|
+
*
|
|
153
|
+
* `payload` is rejected: EventBridge delivers the full matched event JSON as
|
|
154
|
+
* the log message only when the target carries no input transform, and
|
|
155
|
+
* downstream metric filters select on `$.detail.*` fields of that JSON — a
|
|
156
|
+
* transformed message would silently break every selector.
|
|
157
|
+
*/
|
|
158
|
+
function resolveLogGroupTarget(target, messageGroupId, baseProps, targetInput) {
|
|
159
|
+
rejectMessageGroupIdOnNonQueue("logGroup", messageGroupId);
|
|
160
|
+
if (targetInput !== undefined) {
|
|
161
|
+
throw new Error("payload is not supported on log-group event targets — the full matched " +
|
|
162
|
+
"event JSON must reach the log message untransformed so metric-filter " +
|
|
163
|
+
"field selectors keep working.");
|
|
164
|
+
}
|
|
165
|
+
target.addToResourcePolicy(new PolicyStatement({
|
|
166
|
+
effect: Effect.ALLOW,
|
|
167
|
+
actions: ["logs:CreateLogStream", "logs:PutLogEvents"],
|
|
168
|
+
resources: [target.logGroupArn],
|
|
169
|
+
principals: [new ServicePrincipal("events.amazonaws.com")]
|
|
170
|
+
}));
|
|
171
|
+
return {
|
|
172
|
+
bind: (rule) => {
|
|
173
|
+
if (baseProps.deadLetterQueue !== undefined) {
|
|
174
|
+
addToDeadLetterQueueResourcePolicy(rule, baseProps.deadLetterQueue);
|
|
175
|
+
}
|
|
176
|
+
return {
|
|
177
|
+
...bindBaseTargetConfig(baseProps),
|
|
178
|
+
// EventBridge expects the log-group ARN WITHOUT the `:*` suffix that
|
|
179
|
+
// `logGroupArn` carries.
|
|
180
|
+
arn: Stack.of(target).formatArn({
|
|
181
|
+
service: "logs",
|
|
182
|
+
resource: "log-group",
|
|
183
|
+
arnFormat: ArnFormat.COLON_RESOURCE_NAME,
|
|
184
|
+
resourceName: target.logGroupName
|
|
185
|
+
}),
|
|
186
|
+
targetResource: target
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
};
|
|
190
|
+
}
|
|
138
191
|
function rejectMessageGroupIdOnNonQueue(kind, messageGroupId) {
|
|
139
192
|
if (messageGroupId !== undefined) {
|
|
140
193
|
throw new Error(`messageGroupId only applies to FIFO queue targets; received on '${kind}' target`);
|
|
@@ -7,7 +7,7 @@ import { Duration, Tags } from "aws-cdk-lib";
|
|
|
7
7
|
export const APPLICATION_ID_TAG_KEY = "fjall:applicationId";
|
|
8
8
|
export const ALARM_DEFAULTS = {
|
|
9
9
|
EVALUATION_PERIOD: Duration.minutes(5),
|
|
10
|
-
ECS: { CPU: 80, MEMORY: 80, RUNNING_TASKS_MIN: 1 },
|
|
10
|
+
ECS: { CPU: 80, MEMORY: 80, RUNNING_TASKS_MIN: 1, STOPPED_TASKS: 3 },
|
|
11
11
|
ALB: { HTTP_5XX_PERCENT: 5, P99_RESPONSE_TIME_MS: 3000 },
|
|
12
12
|
RDS: { CPU: 80, FREE_STORAGE_GIB: 5, CONNECTIONS: 50 },
|
|
13
13
|
LAMBDA: { ERROR_RATE: 5, DURATION_PERCENT: 80 },
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { Alarm } from "aws-cdk-lib/aws-cloudwatch";
|
|
2
|
+
import type { ICluster } from "aws-cdk-lib/aws-ecs";
|
|
3
|
+
import type { ITopic } from "aws-cdk-lib/aws-sns";
|
|
4
|
+
import type { Construct } from "constructs";
|
|
5
|
+
import { LogGroup } from "../logging/logGroup.js";
|
|
6
|
+
import { Subscription } from "../messaging/subscription.js";
|
|
7
|
+
/**
|
|
8
|
+
* The abnormal-stop allowlist. `EssentialContainerExited` covers every crash
|
|
9
|
+
* after the container started (including crash loops); `TaskFailedToStart`
|
|
10
|
+
* covers image-pull, secret-resolution and dependency-condition failures.
|
|
11
|
+
* Deliberately excludes `ServiceSchedulerInitiated` (scale-in / deploy drain),
|
|
12
|
+
* `UserInitiated` (operator StopTask), `SpotInterruption` and
|
|
13
|
+
* `TerminationNotice` — routine stops that would drown the forensic signal.
|
|
14
|
+
*/
|
|
15
|
+
export declare const ABNORMAL_TASK_STOP_CODES: readonly ["EssentialContainerExited", "TaskFailedToStart"];
|
|
16
|
+
/** Metric emitted into `METRIC_NAMESPACE.ECS` by the watchdog's filter. */
|
|
17
|
+
export declare const TASK_STOP_METRIC_NAME = "StoppedTaskCount";
|
|
18
|
+
export interface EcsTaskStopWatchdogProps {
|
|
19
|
+
scope: Construct;
|
|
20
|
+
/** Cluster name — keys construct ids and the forensic log-group path. */
|
|
21
|
+
clusterName: string;
|
|
22
|
+
/** Cluster whose `clusterArn` scopes the EventBridge pattern. */
|
|
23
|
+
cluster: ICluster;
|
|
24
|
+
/** Steady-state service names to raise per-service churn alarms for. */
|
|
25
|
+
serviceNames: string[];
|
|
26
|
+
/** SNS topic for churn alarms. Omitted → forensic capture only, no alarms. */
|
|
27
|
+
alertsTopic?: ITopic;
|
|
28
|
+
/** Application ID for alarm tagging (webhook-to-application mapping). */
|
|
29
|
+
applicationId?: string;
|
|
30
|
+
}
|
|
31
|
+
export interface EcsTaskStopWatchdog {
|
|
32
|
+
logGroup: LogGroup;
|
|
33
|
+
subscription: Subscription;
|
|
34
|
+
alarms: Alarm[];
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* ECS task-stop watchdog: the forensic artefact for abnormal task stops.
|
|
38
|
+
*
|
|
39
|
+
* A crash-looping service leaves almost no trace once ECS expunges its stopped
|
|
40
|
+
* tasks (~1 hour) — the 2026-08-09 incident produced 69 task deaths with no
|
|
41
|
+
* durable record and no notification. This watchdog captures every abnormal
|
|
42
|
+
* stop event (`stopCode` allowlist above) for one cluster into a 30-day log
|
|
43
|
+
* group `/fjall/<cluster>/task-stops` via a default-bus EventBridge rule, then
|
|
44
|
+
* derives `Fjall/ECS StoppedTaskCount{ClusterName,ServiceName}` through a
|
|
45
|
+
* metric filter and raises a notification-only churn alarm per service.
|
|
46
|
+
*
|
|
47
|
+
* Dimension values are what the raw event carries: `ClusterName` is the full
|
|
48
|
+
* cluster ARN and `ServiceName` is the deployment group (`service:<name>`) —
|
|
49
|
+
* metric-filter dimensions can only select JSON fields, never rewrite them.
|
|
50
|
+
* The filter also drops `family:*` groups (standalone/scheduled tasks), whose
|
|
51
|
+
* stops still land in the log group for forensics but carry no service to
|
|
52
|
+
* alarm on.
|
|
53
|
+
*
|
|
54
|
+
* The rule attaches to the account default bus (AWS service events fire only
|
|
55
|
+
* there), so the `Subscription` is constructed directly with no `eventBus` —
|
|
56
|
+
* the `fromAwsServiceBus` guard exists to stop `aws.*` subscriptions on custom
|
|
57
|
+
* app buses, a misuse this resources-layer composition cannot express.
|
|
58
|
+
*/
|
|
59
|
+
export declare function createEcsTaskStopWatchdog(props: EcsTaskStopWatchdogProps): EcsTaskStopWatchdog;
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { Alarm, ComparisonOperator, Metric, TreatMissingData } from "aws-cdk-lib/aws-cloudwatch";
|
|
2
|
+
import { SnsAction } from "aws-cdk-lib/aws-cloudwatch-actions";
|
|
3
|
+
import { RemovalPolicy } from "aws-cdk-lib";
|
|
4
|
+
import { FilterPattern, MetricFilter, RetentionDays } from "aws-cdk-lib/aws-logs";
|
|
5
|
+
import { ALARM_DEFAULTS, buildAlarmDescription, registerAlarm, tagAlarmsWithApplicationId } from "./alarmDefaults.js";
|
|
6
|
+
import { METRIC_NAMESPACE } from "./metricNamespaces.js";
|
|
7
|
+
import { LogGroup } from "../logging/logGroup.js";
|
|
8
|
+
import { Subscription } from "../messaging/subscription.js";
|
|
9
|
+
/**
|
|
10
|
+
* The abnormal-stop allowlist. `EssentialContainerExited` covers every crash
|
|
11
|
+
* after the container started (including crash loops); `TaskFailedToStart`
|
|
12
|
+
* covers image-pull, secret-resolution and dependency-condition failures.
|
|
13
|
+
* Deliberately excludes `ServiceSchedulerInitiated` (scale-in / deploy drain),
|
|
14
|
+
* `UserInitiated` (operator StopTask), `SpotInterruption` and
|
|
15
|
+
* `TerminationNotice` — routine stops that would drown the forensic signal.
|
|
16
|
+
*/
|
|
17
|
+
export const ABNORMAL_TASK_STOP_CODES = [
|
|
18
|
+
"EssentialContainerExited",
|
|
19
|
+
"TaskFailedToStart"
|
|
20
|
+
];
|
|
21
|
+
/** Metric emitted into `METRIC_NAMESPACE.ECS` by the watchdog's filter. */
|
|
22
|
+
export const TASK_STOP_METRIC_NAME = "StoppedTaskCount";
|
|
23
|
+
/**
|
|
24
|
+
* ECS deployment-group prefix for service tasks. Coupled between the metric
|
|
25
|
+
* filter's pattern and the alarm's ServiceName dimension — drift between the
|
|
26
|
+
* two silently renders every churn alarm dead (metric emitted under one
|
|
27
|
+
* dimension value, alarm querying another).
|
|
28
|
+
*/
|
|
29
|
+
const SERVICE_GROUP_PREFIX = "service:";
|
|
30
|
+
/**
|
|
31
|
+
* ECS task-stop watchdog: the forensic artefact for abnormal task stops.
|
|
32
|
+
*
|
|
33
|
+
* A crash-looping service leaves almost no trace once ECS expunges its stopped
|
|
34
|
+
* tasks (~1 hour) — the 2026-08-09 incident produced 69 task deaths with no
|
|
35
|
+
* durable record and no notification. This watchdog captures every abnormal
|
|
36
|
+
* stop event (`stopCode` allowlist above) for one cluster into a 30-day log
|
|
37
|
+
* group `/fjall/<cluster>/task-stops` via a default-bus EventBridge rule, then
|
|
38
|
+
* derives `Fjall/ECS StoppedTaskCount{ClusterName,ServiceName}` through a
|
|
39
|
+
* metric filter and raises a notification-only churn alarm per service.
|
|
40
|
+
*
|
|
41
|
+
* Dimension values are what the raw event carries: `ClusterName` is the full
|
|
42
|
+
* cluster ARN and `ServiceName` is the deployment group (`service:<name>`) —
|
|
43
|
+
* metric-filter dimensions can only select JSON fields, never rewrite them.
|
|
44
|
+
* The filter also drops `family:*` groups (standalone/scheduled tasks), whose
|
|
45
|
+
* stops still land in the log group for forensics but carry no service to
|
|
46
|
+
* alarm on.
|
|
47
|
+
*
|
|
48
|
+
* The rule attaches to the account default bus (AWS service events fire only
|
|
49
|
+
* there), so the `Subscription` is constructed directly with no `eventBus` —
|
|
50
|
+
* the `fromAwsServiceBus` guard exists to stop `aws.*` subscriptions on custom
|
|
51
|
+
* app buses, a misuse this resources-layer composition cannot express.
|
|
52
|
+
*/
|
|
53
|
+
export function createEcsTaskStopWatchdog(props) {
|
|
54
|
+
const { scope, clusterName, cluster, serviceNames, alertsTopic, applicationId } = props;
|
|
55
|
+
// Fixed-name log group: the env-aware RETAIN default would orphan
|
|
56
|
+
// `/fjall/<cluster>/task-stops` on stack delete or a `taskStopWatchdog:
|
|
57
|
+
// false` toggle, and the next synth's CREATE then collides with the
|
|
58
|
+
// orphan. Rolling 30-day telemetry is not worth bricking redeploys.
|
|
59
|
+
const logGroup = new LogGroup(scope, `${clusterName}TaskStopsLogGroup`, {
|
|
60
|
+
logGroupName: `/fjall/${clusterName}/task-stops`,
|
|
61
|
+
retention: RetentionDays.ONE_MONTH,
|
|
62
|
+
removalPolicy: RemovalPolicy.DESTROY
|
|
63
|
+
});
|
|
64
|
+
const subscription = new Subscription(scope, `${clusterName}TaskStops`, {
|
|
65
|
+
description: `Forensic capture of abnormal task stops in ECS cluster ${clusterName}`,
|
|
66
|
+
pattern: {
|
|
67
|
+
source: ["aws.ecs"],
|
|
68
|
+
detailType: ["ECS Task State Change"],
|
|
69
|
+
detail: {
|
|
70
|
+
clusterArn: [cluster.clusterArn],
|
|
71
|
+
lastStatus: ["STOPPED"],
|
|
72
|
+
stopCode: [...ABNORMAL_TASK_STOP_CODES]
|
|
73
|
+
}
|
|
74
|
+
},
|
|
75
|
+
target: logGroup
|
|
76
|
+
});
|
|
77
|
+
new MetricFilter(scope, `${clusterName}TaskStopsMetricFilter`, {
|
|
78
|
+
logGroup,
|
|
79
|
+
metricNamespace: METRIC_NAMESPACE.ECS,
|
|
80
|
+
metricName: TASK_STOP_METRIC_NAME,
|
|
81
|
+
filterPattern: FilterPattern.literal(`{ $.detail.group = "${SERVICE_GROUP_PREFIX}*" }`),
|
|
82
|
+
metricValue: "1",
|
|
83
|
+
dimensions: {
|
|
84
|
+
ClusterName: "$.detail.clusterArn",
|
|
85
|
+
ServiceName: "$.detail.group"
|
|
86
|
+
}
|
|
87
|
+
});
|
|
88
|
+
const alarms = [];
|
|
89
|
+
if (alertsTopic === undefined) {
|
|
90
|
+
return { logGroup, subscription, alarms };
|
|
91
|
+
}
|
|
92
|
+
const snsAction = new SnsAction(alertsTopic);
|
|
93
|
+
for (const serviceName of serviceNames) {
|
|
94
|
+
const alarm = new Alarm(scope, `${serviceName}TaskStopsAlarm`, {
|
|
95
|
+
alarmDescription: buildAlarmDescription(`ECS service ${serviceName} abnormal task stops — crash loop or failing deployment`, applicationId),
|
|
96
|
+
metric: new Metric({
|
|
97
|
+
namespace: METRIC_NAMESPACE.ECS,
|
|
98
|
+
metricName: TASK_STOP_METRIC_NAME,
|
|
99
|
+
period: ALARM_DEFAULTS.EVALUATION_PERIOD,
|
|
100
|
+
statistic: "Sum",
|
|
101
|
+
dimensionsMap: {
|
|
102
|
+
ClusterName: cluster.clusterArn,
|
|
103
|
+
ServiceName: `${SERVICE_GROUP_PREFIX}${serviceName}`
|
|
104
|
+
}
|
|
105
|
+
}),
|
|
106
|
+
threshold: ALARM_DEFAULTS.ECS.STOPPED_TASKS,
|
|
107
|
+
evaluationPeriods: 1,
|
|
108
|
+
comparisonOperator: ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
|
|
109
|
+
treatMissingData: TreatMissingData.NOT_BREACHING
|
|
110
|
+
});
|
|
111
|
+
registerAlarm(alarm, snsAction, alarms);
|
|
112
|
+
}
|
|
113
|
+
tagAlarmsWithApplicationId(alarms, applicationId);
|
|
114
|
+
return { logGroup, subscription, alarms };
|
|
115
|
+
}
|
|
@@ -7,4 +7,5 @@ export { createSqsDlqAlarms, type SqsAlarmThresholds, type SqsDlqAlarmsProps } f
|
|
|
7
7
|
export { createClickHouseAlarms, type ClickHouseAlarmThresholds, type ClickHouseAlarmsProps } from "./clickhouseAlarms.js";
|
|
8
8
|
export { createBuildkiteAlarms, type BuildkiteAlarmsProps } from "./buildkiteAlarms.js";
|
|
9
9
|
export { createLogPatternAlarms, type LogPatternAlarmSpec, type LogPatternAlarmsProps } from "./logPatternAlarms.js";
|
|
10
|
+
export { createEcsTaskStopWatchdog, ABNORMAL_TASK_STOP_CODES, TASK_STOP_METRIC_NAME, type EcsTaskStopWatchdog, type EcsTaskStopWatchdogProps } from "./ecsTaskStopWatchdog.js";
|
|
10
11
|
export { METRIC_NAMESPACE, type MetricNamespace } from "./metricNamespaces.js";
|
|
@@ -7,4 +7,5 @@ export { createSqsDlqAlarms } from "./sqsAlarms.js";
|
|
|
7
7
|
export { createClickHouseAlarms } from "./clickhouseAlarms.js";
|
|
8
8
|
export { createBuildkiteAlarms } from "./buildkiteAlarms.js";
|
|
9
9
|
export { createLogPatternAlarms } from "./logPatternAlarms.js";
|
|
10
|
+
export { createEcsTaskStopWatchdog, ABNORMAL_TASK_STOP_CODES, TASK_STOP_METRIC_NAME } from "./ecsTaskStopWatchdog.js";
|
|
10
11
|
export { METRIC_NAMESPACE } from "./metricNamespaces.js";
|