@fjall/components-infrastructure 39.0.5 → 39.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/patterns/aws/computeEcs.d.ts +65 -21
- package/dist/lib/patterns/aws/computeEcs.js +243 -95
- package/dist/lib/patterns/aws/computeEcsTypes.d.ts +49 -25
- package/dist/lib/resources/aws/compute/ecs.d.ts +3 -0
- package/dist/lib/resources/aws/compute/ecs.js +4 -0
- package/dist/lib/resources/aws/compute/ecsLifecycleHookMigration.d.ts +32 -62
- package/dist/lib/resources/aws/compute/ecsLifecycleHookMigration.js +96 -50
- package/dist/lib/resources/aws/compute/ecsRolloutBarrier.d.ts +61 -0
- package/dist/lib/resources/aws/compute/ecsRolloutBarrier.js +185 -0
- package/dist/lib/resources/aws/compute/ecsRolloutBarrierLambda.source.cjs +475 -0
- package/dist/lib/resources/aws/compute/ecsTaskDefinition.d.ts +19 -8
- package/dist/lib/resources/aws/compute/ecsTaskDefinition.js +16 -7
- package/dist/lib/resources/aws/compute/ecsValidation.d.ts +3 -3
- package/dist/lib/resources/aws/compute/ecsValidation.js +3 -3
- package/dist/lib/resources/aws/compute/lambda.d.ts +6 -0
- package/dist/lib/resources/aws/compute/lambda.js +18 -12
- package/dist/lib/resources/aws/compute/lifecycleHookLambda.source.cjs +329 -82
- package/package.json +5 -4
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { readFileSync } from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
5
|
+
import { ECS_ROLLOUT_BARRIER_RESOURCE_TYPE } from "@fjall/util/aws";
|
|
6
|
+
import { ArnFormat, Aws, CustomResource, Duration, Fn, RemovalPolicy, Stack, Token } from "aws-cdk-lib";
|
|
7
|
+
import { PolicyStatement } from "aws-cdk-lib/aws-iam";
|
|
8
|
+
import { Code, Runtime } from "aws-cdk-lib/aws-lambda";
|
|
9
|
+
import { LogLevel } from "aws-cdk-lib/aws-stepfunctions";
|
|
10
|
+
import { Construct } from "constructs";
|
|
11
|
+
import { markDeployTimeWriter } from "../../../utils/resourceTypeTags.js";
|
|
12
|
+
import { DEFAULT_FRAMEWORK_LOG_RETENTION, LogGroup } from "../logging/logGroup.js";
|
|
13
|
+
import { CustomResourceProvider } from "../utilities/customResourceProvider.js";
|
|
14
|
+
import { LambdaFunction } from "./lambda.js";
|
|
15
|
+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
16
|
+
const LAMBDA_SOURCE_FILE = "ecsRolloutBarrierLambda.source.cjs";
|
|
17
|
+
/** How often the provider asks `isComplete` whether the rollout has finished. */
|
|
18
|
+
const QUERY_INTERVAL = Duration.seconds(15);
|
|
19
|
+
/**
|
|
20
|
+
* The provider's ceiling, CloudFormation's own for a custom resource. Limit
|
|
21
|
+
* (2026-10-01): a rollout longer than an hour fails the barrier; the engine's
|
|
22
|
+
* own deploy budget is 30 minutes.
|
|
23
|
+
*/
|
|
24
|
+
const TOTAL_TIMEOUT = Duration.hours(1);
|
|
25
|
+
/** One handler call: a DescribeServices, then at most a few listing pages. */
|
|
26
|
+
const HANDLER_TIMEOUT_SECONDS = 60;
|
|
27
|
+
const PROVIDER_ID = "FjallEcsRolloutBarrierProvider";
|
|
28
|
+
const WAITER_LOG_GROUP_ID = "WaiterLogGroup";
|
|
29
|
+
/** Hex characters of the construct path's digest the waiter group's name carries. */
|
|
30
|
+
const WAITER_NAME_PATH_DIGEST_LENGTH = 24;
|
|
31
|
+
/**
|
|
32
|
+
* `/aws/vendedlogs/states/<stack name>-<stack id>-<path digest>`, the name
|
|
33
|
+
* main's `vendedLogsLogGroup` gives the same group, so a 40.0.0 upgrade keeps
|
|
34
|
+
* it in place. CloudWatch Logs authorises Step Functions' delivery to every
|
|
35
|
+
* group under `/aws/vendedlogs/` through one statement of the account's
|
|
36
|
+
* resource policy, where a group outside it adds its own ARN to that shared
|
|
37
|
+
* policy. The stack's id differs for every stack ever created under its
|
|
38
|
+
* name, and the digest of the construct path makes the name unique within
|
|
39
|
+
* the stack.
|
|
40
|
+
*/
|
|
41
|
+
function waiterLogGroupName(scope) {
|
|
42
|
+
const stackId = Fn.select(2, Fn.split("/", Aws.STACK_ID));
|
|
43
|
+
const pathDigest = createHash("sha256")
|
|
44
|
+
.update(`${scope.node.path}/${WAITER_LOG_GROUP_ID}`)
|
|
45
|
+
.digest("hex")
|
|
46
|
+
.slice(0, WAITER_NAME_PATH_DIGEST_LENGTH);
|
|
47
|
+
return `/aws/vendedlogs/states/${Aws.STACK_NAME}-${stackId}-${pathDigest}`;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* The stack's one rollout-barrier provider: the `onEvent` and `isComplete`
|
|
51
|
+
* handlers, from one inline source, and the provider framework that asks
|
|
52
|
+
* `isComplete` every 15 s for at most an hour. Each barrier grants the
|
|
53
|
+
* `isComplete` role the reads of its own service only; the role also reads
|
|
54
|
+
* its own stack's status, to complete at once on a rollback.
|
|
55
|
+
*
|
|
56
|
+
* The framework's waiter state machine logs at FATAL, without execution
|
|
57
|
+
* data, to a group named under `/aws/vendedlogs/states/`
|
|
58
|
+
* (`waiterLogGroupName`). Being named, the group is destroyed with its stack
|
|
59
|
+
* whatever the stage: a group a rollback retained would collide with the one
|
|
60
|
+
* the next upgrade creates under the same name.
|
|
61
|
+
*/
|
|
62
|
+
export class EcsRolloutBarrierProvider extends Construct {
|
|
63
|
+
/** The stack's provider, made on first use. */
|
|
64
|
+
static of(scope) {
|
|
65
|
+
const stack = Stack.of(scope);
|
|
66
|
+
const existing = stack.node.tryFindChild(PROVIDER_ID);
|
|
67
|
+
if (existing instanceof EcsRolloutBarrierProvider)
|
|
68
|
+
return existing;
|
|
69
|
+
if (existing !== undefined) {
|
|
70
|
+
throw new Error(`${stack.node.path}/${PROVIDER_ID} is taken by a construct that is not the rollout-barrier provider`);
|
|
71
|
+
}
|
|
72
|
+
return new EcsRolloutBarrierProvider(stack, PROVIDER_ID);
|
|
73
|
+
}
|
|
74
|
+
serviceToken;
|
|
75
|
+
isCompleteHandler;
|
|
76
|
+
granted = new Set();
|
|
77
|
+
constructor(scope, id) {
|
|
78
|
+
super(scope, id);
|
|
79
|
+
// One source serves both handlers, each by its own export.
|
|
80
|
+
const code = Code.fromInline(readFileSync(path.resolve(__dirname, LAMBDA_SOURCE_FILE), "utf-8"));
|
|
81
|
+
const handlerProps = (handler, description) => ({
|
|
82
|
+
runtime: Runtime.NODEJS_24_X,
|
|
83
|
+
handler,
|
|
84
|
+
stackOutputs: "none",
|
|
85
|
+
code,
|
|
86
|
+
lambdaDescription: description,
|
|
87
|
+
roleDescription: `Execution role for the Fjall ECS rollout barrier (${handler})`,
|
|
88
|
+
timeout: HANDLER_TIMEOUT_SECONDS,
|
|
89
|
+
memorySize: 256,
|
|
90
|
+
inlinePolicy: []
|
|
91
|
+
});
|
|
92
|
+
const onEvent = new LambdaFunction(this, "OnEvent", handlerProps("index.onEvent", "Fjall ECS rollout barrier: records when the wait began"));
|
|
93
|
+
this.isCompleteHandler = new LambdaFunction(this, "IsComplete", handlerProps("index.isComplete", "Fjall ECS rollout barrier: reads whether the service's deployment finished"));
|
|
94
|
+
const provider = new CustomResourceProvider(this, "Provider", {
|
|
95
|
+
onEventHandler: onEvent,
|
|
96
|
+
isCompleteHandler: this.isCompleteHandler,
|
|
97
|
+
queryInterval: QUERY_INTERVAL,
|
|
98
|
+
totalTimeout: TOTAL_TIMEOUT,
|
|
99
|
+
logGroup: new LogGroup(this, "ProviderLogGroup", {
|
|
100
|
+
retention: DEFAULT_FRAMEWORK_LOG_RETENTION
|
|
101
|
+
}),
|
|
102
|
+
disableWaiterStateMachineLogging: false,
|
|
103
|
+
waiterStateMachineLogOptions: {
|
|
104
|
+
destination: new LogGroup(this, WAITER_LOG_GROUP_ID, {
|
|
105
|
+
logGroupName: waiterLogGroupName(this),
|
|
106
|
+
retention: DEFAULT_FRAMEWORK_LOG_RETENTION,
|
|
107
|
+
removalPolicy: RemovalPolicy.DESTROY
|
|
108
|
+
}),
|
|
109
|
+
level: LogLevel.FATAL,
|
|
110
|
+
includeExecutionData: false
|
|
111
|
+
}
|
|
112
|
+
});
|
|
113
|
+
this.serviceToken = provider.serviceToken;
|
|
114
|
+
this.isCompleteHandler.addToRolePolicy(new PolicyStatement({
|
|
115
|
+
actions: ["cloudformation:DescribeStacks"],
|
|
116
|
+
resources: [Aws.STACK_ID]
|
|
117
|
+
}));
|
|
118
|
+
markDeployTimeWriter(this);
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* The reads `isComplete` makes of one service, scoped to it: its
|
|
122
|
+
* description, its deployments and their revisions (AWS service reference,
|
|
123
|
+
* ecs.json, 2026-10-01). Granted once a service.
|
|
124
|
+
*
|
|
125
|
+
* The ARNs are built from the literal cluster and service names, never from
|
|
126
|
+
* the service's attributes: the grant lands on the one role every barrier in
|
|
127
|
+
* the stack shares, so a reference to a service would make each barrier
|
|
128
|
+
* wait for every owner, and an owner that itself awaits another owner would
|
|
129
|
+
* close a cycle.
|
|
130
|
+
*/
|
|
131
|
+
grantRolloutReads(clusterName, serviceName) {
|
|
132
|
+
if (Token.isUnresolved(clusterName) || Token.isUnresolved(serviceName)) {
|
|
133
|
+
throw new Error(`${this.node.path}: a rollout barrier's grant needs the literal cluster and service names`);
|
|
134
|
+
}
|
|
135
|
+
const key = `${clusterName}/${serviceName}`;
|
|
136
|
+
if (this.granted.has(key))
|
|
137
|
+
return;
|
|
138
|
+
this.granted.add(key);
|
|
139
|
+
const stack = Stack.of(this);
|
|
140
|
+
const arnOf = (resource, resourceName) => stack.formatArn({
|
|
141
|
+
service: "ecs",
|
|
142
|
+
resource,
|
|
143
|
+
resourceName,
|
|
144
|
+
arnFormat: ArnFormat.SLASH_RESOURCE_NAME
|
|
145
|
+
});
|
|
146
|
+
this.isCompleteHandler.addToRolePolicy(new PolicyStatement({
|
|
147
|
+
actions: [
|
|
148
|
+
"ecs:DescribeServices",
|
|
149
|
+
"ecs:ListServiceDeployments",
|
|
150
|
+
"ecs:DescribeServiceDeployments",
|
|
151
|
+
"ecs:DescribeServiceRevisions"
|
|
152
|
+
],
|
|
153
|
+
resources: [
|
|
154
|
+
arnOf("service", key),
|
|
155
|
+
arnOf("service-deployment", `${key}/*`),
|
|
156
|
+
arnOf("service-revision", `${key}/*`)
|
|
157
|
+
]
|
|
158
|
+
}));
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* A custom resource that completes only once ECS reports the service's
|
|
163
|
+
* deployment of its task definition finished, and fails once ECS rolled it
|
|
164
|
+
* back (FJL-901). Its dependants therefore wait for the rollout, PRE_SCALE_UP
|
|
165
|
+
* migrate hook included, whatever CloudFormation's own stabilisation of the
|
|
166
|
+
* service reports. Delete only acknowledges.
|
|
167
|
+
*/
|
|
168
|
+
export class EcsRolloutBarrier extends Construct {
|
|
169
|
+
resource;
|
|
170
|
+
constructor(scope, id, props) {
|
|
171
|
+
super(scope, id);
|
|
172
|
+
const provider = EcsRolloutBarrierProvider.of(this);
|
|
173
|
+
provider.grantRolloutReads(props.clusterName, props.serviceName);
|
|
174
|
+
this.resource = new CustomResource(this, "Resource", {
|
|
175
|
+
serviceToken: provider.serviceToken,
|
|
176
|
+
resourceType: ECS_ROLLOUT_BARRIER_RESOURCE_TYPE,
|
|
177
|
+
properties: {
|
|
178
|
+
Cluster: props.clusterName,
|
|
179
|
+
Service: props.service.serviceArn,
|
|
180
|
+
TaskDefinition: props.taskDefinitionArn
|
|
181
|
+
}
|
|
182
|
+
});
|
|
183
|
+
this.resource.node.addDependency(props.service);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
@@ -0,0 +1,475 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ECS rollout barrier (FJL-901). Inlined into the onEvent and isComplete
|
|
3
|
+
* functions of a custom-resource provider by ecsRolloutBarrier.ts.
|
|
4
|
+
*
|
|
5
|
+
* CloudFormation starts a resource's dependants when the resource logs
|
|
6
|
+
* "Eventual consistency check initiated", and since 2026-10-01 an updating
|
|
7
|
+
* ECS service logs it long before its deployment, PRE_SCALE_UP migrate hook
|
|
8
|
+
* included, has finished. A custom resource logs no such check: its
|
|
9
|
+
* dependants wait for its response. So this one waits, reading ECS, until the
|
|
10
|
+
* service's deployment of the task definition its properties name has
|
|
11
|
+
* finished, and fails once ECS has rolled that deployment back, which fails
|
|
12
|
+
* the stack's update.
|
|
13
|
+
*
|
|
14
|
+
* The verdict reads the service's live state first, so the same code is right
|
|
15
|
+
* on the forward path and when the service needed no new deployment. On a
|
|
16
|
+
* stack rollback (CloudFormation re-sends the previous task definition) the
|
|
17
|
+
* barrier completes at once: it has nothing to order there. When ECS lists no
|
|
18
|
+
* deployment of the task definition, the live state decides alone: a service
|
|
19
|
+
* at rest on that revision, or on a later one of its family (a restart's or a
|
|
20
|
+
* code-only deploy's copy), finished that rollout before the barrier began.
|
|
21
|
+
*
|
|
22
|
+
* Only a definitive verdict fails: the provider framework submits FAILED for
|
|
23
|
+
* any error isComplete throws, so a read that failed for a transient reason
|
|
24
|
+
* (a throttle, a 5xx, a network fault, this invocation's own deadline) waits
|
|
25
|
+
* for the next poll instead, and says so in the log. So does an access denial
|
|
26
|
+
* in the wait's first two minutes, while IAM propagates the barrier's grant.
|
|
27
|
+
*
|
|
28
|
+
* CommonJS because Code.fromInline lands the source as `index.js`, which
|
|
29
|
+
* Lambda treats as CommonJS by default.
|
|
30
|
+
*/
|
|
31
|
+
const {
|
|
32
|
+
ECSClient,
|
|
33
|
+
DescribeServicesCommand,
|
|
34
|
+
ListServiceDeploymentsCommand,
|
|
35
|
+
DescribeServiceDeploymentsCommand,
|
|
36
|
+
DescribeServiceRevisionsCommand
|
|
37
|
+
} = require("@aws-sdk/client-ecs");
|
|
38
|
+
const {
|
|
39
|
+
CloudFormationClient,
|
|
40
|
+
DescribeStacksCommand
|
|
41
|
+
} = require("@aws-sdk/client-cloudformation");
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* How long the barrier waits for a deployment of its task definition to be
|
|
45
|
+
* listed, or for the service to come to rest on it.
|
|
46
|
+
*/
|
|
47
|
+
const NO_DEPLOYMENT_GRACE_MS = 10 * 60 * 1000;
|
|
48
|
+
|
|
49
|
+
/** How far back the barrier lists the service's deployments. */
|
|
50
|
+
const LISTING_WINDOW_MS = 6 * 60 * 60 * 1000;
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* One isComplete invocation's reads, well inside the function's 60 s timeout,
|
|
54
|
+
* which the framework would otherwise read as a failure.
|
|
55
|
+
*/
|
|
56
|
+
const INVOCATION_BUDGET_MS = 40 * 1000;
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Every service-deployment status ECS reports (ServiceDeploymentStatus,
|
|
60
|
+
* @aws-sdk/client-ecs 3.1101.0), by what it means for the barrier. A status
|
|
61
|
+
* outside all three waits, and the verdict log names it: an unknown status
|
|
62
|
+
* never completes the barrier.
|
|
63
|
+
*/
|
|
64
|
+
const SUCCEEDED = new Set(["SUCCESSFUL"]);
|
|
65
|
+
const FAILED = new Set(["ROLLBACK_SUCCESSFUL", "ROLLBACK_FAILED", "STOPPED"]);
|
|
66
|
+
const WAITING = new Set([
|
|
67
|
+
"PENDING",
|
|
68
|
+
"IN_PROGRESS",
|
|
69
|
+
"STOP_REQUESTED",
|
|
70
|
+
"ROLLBACK_REQUESTED",
|
|
71
|
+
"ROLLBACK_IN_PROGRESS"
|
|
72
|
+
]);
|
|
73
|
+
|
|
74
|
+
/** The stack status under which CloudFormation rolls a failed update back. */
|
|
75
|
+
const ROLLING_BACK = "UPDATE_ROLLBACK_IN_PROGRESS";
|
|
76
|
+
|
|
77
|
+
/** The most deployments or revisions one Describe call takes. */
|
|
78
|
+
const DESCRIBE_BATCH = 20;
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Failures that say nothing about the rollout: the read is retried at the
|
|
82
|
+
* next poll. Any other failure (access denied, a service that no longer
|
|
83
|
+
* exists, a malformed request) is definitive and fails the barrier.
|
|
84
|
+
*/
|
|
85
|
+
const TRANSIENT_ERROR_NAMES = new Set([
|
|
86
|
+
"AbortError",
|
|
87
|
+
"Throttling",
|
|
88
|
+
"ThrottlingException",
|
|
89
|
+
"TooManyRequestsException",
|
|
90
|
+
"RequestLimitExceeded",
|
|
91
|
+
"ServerException",
|
|
92
|
+
"ServiceUnavailableException",
|
|
93
|
+
"InternalFailure",
|
|
94
|
+
"InternalServerError",
|
|
95
|
+
"RequestTimeout",
|
|
96
|
+
"RequestTimeoutException",
|
|
97
|
+
"TimeoutError"
|
|
98
|
+
]);
|
|
99
|
+
const TRANSIENT_ERROR_CODES = new Set([
|
|
100
|
+
"ECONNRESET",
|
|
101
|
+
"ECONNREFUSED",
|
|
102
|
+
"ETIMEDOUT",
|
|
103
|
+
"EPIPE",
|
|
104
|
+
"EAI_AGAIN",
|
|
105
|
+
"ENOTFOUND"
|
|
106
|
+
]);
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* How long after the wait began an access denial still reads as IAM
|
|
110
|
+
* propagating the barrier's grant, which lands on the shared role in the same
|
|
111
|
+
* stack operation. After it, a denial is definitive.
|
|
112
|
+
*/
|
|
113
|
+
const ACCESS_PROPAGATION_GRACE_MS = 2 * 60 * 1000;
|
|
114
|
+
const ACCESS_DENIED_NAMES = new Set(["AccessDeniedException", "AccessDenied"]);
|
|
115
|
+
|
|
116
|
+
function isAccessDenied(err) {
|
|
117
|
+
return (
|
|
118
|
+
err !== null && typeof err === "object" && ACCESS_DENIED_NAMES.has(err.name)
|
|
119
|
+
);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function isTransient(err) {
|
|
123
|
+
if (err === null || typeof err !== "object") return false;
|
|
124
|
+
if (TRANSIENT_ERROR_NAMES.has(err.name)) return true;
|
|
125
|
+
if (TRANSIENT_ERROR_CODES.has(err.code)) return true;
|
|
126
|
+
const status = err.$metadata && err.$metadata.httpStatusCode;
|
|
127
|
+
return typeof status === "number" && status >= 500;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
let defaultClient;
|
|
131
|
+
function getDefaultClient() {
|
|
132
|
+
if (!defaultClient) defaultClient = new ECSClient({});
|
|
133
|
+
return defaultClient;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
let defaultStackClient;
|
|
137
|
+
function getDefaultStackClient() {
|
|
138
|
+
if (!defaultStackClient) defaultStackClient = new CloudFormationClient({});
|
|
139
|
+
return defaultStackClient;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Whether CloudFormation is rolling the barrier's stack back. On a rollback
|
|
144
|
+
* the barrier has nothing to order: its dependants return to the code that
|
|
145
|
+
* ran before, whatever the order, and a barrier that waited could only fail
|
|
146
|
+
* the rollback, whichever order CloudFormation rolls resources back in.
|
|
147
|
+
*/
|
|
148
|
+
async function stackRollingBack(stackClient, stackId, abortSignal) {
|
|
149
|
+
const answer = await stackClient.send(
|
|
150
|
+
new DescribeStacksCommand({ StackName: stackId }),
|
|
151
|
+
{ abortSignal }
|
|
152
|
+
);
|
|
153
|
+
const stack = (answer.Stacks || [])[0];
|
|
154
|
+
if (stack === undefined) {
|
|
155
|
+
throw new Error(`CloudFormation answered with no stack ${stackId}`);
|
|
156
|
+
}
|
|
157
|
+
return stack.StackStatus === ROLLING_BACK;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function propertiesOf(event) {
|
|
161
|
+
const properties = (event && event.ResourceProperties) || {};
|
|
162
|
+
const { Cluster, Service, TaskDefinition } = properties;
|
|
163
|
+
if (
|
|
164
|
+
typeof Cluster !== "string" ||
|
|
165
|
+
Cluster === "" ||
|
|
166
|
+
typeof Service !== "string" ||
|
|
167
|
+
Service === "" ||
|
|
168
|
+
typeof TaskDefinition !== "string" ||
|
|
169
|
+
TaskDefinition === ""
|
|
170
|
+
) {
|
|
171
|
+
throw new Error(
|
|
172
|
+
"The rollout barrier needs its Cluster, Service and TaskDefinition properties"
|
|
173
|
+
);
|
|
174
|
+
}
|
|
175
|
+
return { cluster: Cluster, service: Service, taskDefinition: TaskDefinition };
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function physicalIdOf(event, properties) {
|
|
179
|
+
return (
|
|
180
|
+
(event && event.PhysicalResourceId) ||
|
|
181
|
+
`rollout-barrier:${properties.cluster}:${properties.service}`
|
|
182
|
+
);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
async function onEvent(event) {
|
|
186
|
+
if (event.RequestType === "Delete") {
|
|
187
|
+
return { PhysicalResourceId: event.PhysicalResourceId };
|
|
188
|
+
}
|
|
189
|
+
const properties = propertiesOf(event);
|
|
190
|
+
return {
|
|
191
|
+
PhysicalResourceId: physicalIdOf(event, properties),
|
|
192
|
+
Data: { WaitStartedAt: String(Date.now()) }
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/** The service's deployments, as DescribeServices answers them now. */
|
|
197
|
+
async function liveDeployments(client, properties, abortSignal) {
|
|
198
|
+
const answer = await client.send(
|
|
199
|
+
new DescribeServicesCommand({
|
|
200
|
+
cluster: properties.cluster,
|
|
201
|
+
services: [properties.service]
|
|
202
|
+
}),
|
|
203
|
+
{ abortSignal }
|
|
204
|
+
);
|
|
205
|
+
const service = (answer.services || [])[0];
|
|
206
|
+
if (service === undefined) {
|
|
207
|
+
throw new Error(
|
|
208
|
+
`ECS answered with no service ${properties.service} in ${properties.cluster}`
|
|
209
|
+
);
|
|
210
|
+
}
|
|
211
|
+
return service.deployments || [];
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
function primaryOf(deployments) {
|
|
215
|
+
return deployments.find((deployment) => deployment.status === "PRIMARY");
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* Whether the PRIMARY deployment runs `taskDefinition`, has completed, and
|
|
220
|
+
* runs every task it wants.
|
|
221
|
+
*/
|
|
222
|
+
function liveRolloutComplete(deployments, properties) {
|
|
223
|
+
const primary = primaryOf(deployments);
|
|
224
|
+
return (
|
|
225
|
+
primary !== undefined &&
|
|
226
|
+
primary.taskDefinition === properties.taskDefinition &&
|
|
227
|
+
primary.rolloutState === "COMPLETED" &&
|
|
228
|
+
primary.runningCount === primary.desiredCount
|
|
229
|
+
);
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/** A task-definition ARN's family and revision, or undefined when it has neither. */
|
|
233
|
+
function familyRevisionOf(taskDefinitionArn) {
|
|
234
|
+
const match = /:task-definition\/([^:/]+):(\d+)$/.exec(
|
|
235
|
+
typeof taskDefinitionArn === "string" ? taskDefinitionArn : ""
|
|
236
|
+
);
|
|
237
|
+
return match === null
|
|
238
|
+
? undefined
|
|
239
|
+
: { family: match[1], revision: Number(match[2]) };
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* Whether the service is at rest on `taskDefinition` or on a later revision
|
|
244
|
+
* of its family: its PRIMARY deployment has completed, and no other
|
|
245
|
+
* deployment is still active. Revisions only grow, so a later one was
|
|
246
|
+
* registered after it, which a restart's or a code-only deploy's copy is;
|
|
247
|
+
* while ECS still shows an earlier revision, the service is not at rest on
|
|
248
|
+
* this one, whatever it shows next. Task counts are not read: the rollout
|
|
249
|
+
* finished, and a shortfall since is capacity, not order.
|
|
250
|
+
*/
|
|
251
|
+
function atRestOnOrAfter(deployments, taskDefinition) {
|
|
252
|
+
const primary = primaryOf(deployments);
|
|
253
|
+
if (primary === undefined || primary.rolloutState !== "COMPLETED") {
|
|
254
|
+
return false;
|
|
255
|
+
}
|
|
256
|
+
if (deployments.some((deployment) => deployment.status === "ACTIVE")) {
|
|
257
|
+
return false;
|
|
258
|
+
}
|
|
259
|
+
const wanted = familyRevisionOf(taskDefinition);
|
|
260
|
+
const live = familyRevisionOf(primary.taskDefinition);
|
|
261
|
+
return (
|
|
262
|
+
wanted !== undefined &&
|
|
263
|
+
live !== undefined &&
|
|
264
|
+
live.family === wanted.family &&
|
|
265
|
+
live.revision >= wanted.revision
|
|
266
|
+
);
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
async function inBatches(items, read) {
|
|
270
|
+
const out = [];
|
|
271
|
+
for (let at = 0; at < items.length; at += DESCRIBE_BATCH) {
|
|
272
|
+
out.push(...(await read(items.slice(at, at + DESCRIBE_BATCH))));
|
|
273
|
+
}
|
|
274
|
+
return out;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* The newest service deployment whose target revision runs `taskDefinition`,
|
|
279
|
+
* or undefined when none is listed yet.
|
|
280
|
+
*/
|
|
281
|
+
async function newestDeploymentOf(client, properties, now, abortSignal) {
|
|
282
|
+
const arns = [];
|
|
283
|
+
let nextToken;
|
|
284
|
+
do {
|
|
285
|
+
const page = await client.send(
|
|
286
|
+
new ListServiceDeploymentsCommand({
|
|
287
|
+
cluster: properties.cluster,
|
|
288
|
+
service: properties.service,
|
|
289
|
+
createdAt: { after: new Date(now - LISTING_WINDOW_MS) },
|
|
290
|
+
...(nextToken !== undefined ? { nextToken } : {})
|
|
291
|
+
}),
|
|
292
|
+
{ abortSignal }
|
|
293
|
+
);
|
|
294
|
+
for (const summary of page.serviceDeployments || []) {
|
|
295
|
+
if (summary.serviceDeploymentArn) arns.push(summary.serviceDeploymentArn);
|
|
296
|
+
}
|
|
297
|
+
nextToken = page.nextToken;
|
|
298
|
+
} while (nextToken);
|
|
299
|
+
if (arns.length === 0) return undefined;
|
|
300
|
+
|
|
301
|
+
const deployments = await inBatches(arns, async (batch) => {
|
|
302
|
+
const answer = await client.send(
|
|
303
|
+
new DescribeServiceDeploymentsCommand({ serviceDeploymentArns: batch }),
|
|
304
|
+
{ abortSignal }
|
|
305
|
+
);
|
|
306
|
+
return answer.serviceDeployments || [];
|
|
307
|
+
});
|
|
308
|
+
const revisionArns = [
|
|
309
|
+
...new Set(
|
|
310
|
+
deployments
|
|
311
|
+
.map(
|
|
312
|
+
(deployment) =>
|
|
313
|
+
deployment.targetServiceRevision &&
|
|
314
|
+
deployment.targetServiceRevision.arn
|
|
315
|
+
)
|
|
316
|
+
.filter((arn) => typeof arn === "string")
|
|
317
|
+
)
|
|
318
|
+
];
|
|
319
|
+
const revisions = await inBatches(revisionArns, async (batch) => {
|
|
320
|
+
const answer = await client.send(
|
|
321
|
+
new DescribeServiceRevisionsCommand({ serviceRevisionArns: batch }),
|
|
322
|
+
{ abortSignal }
|
|
323
|
+
);
|
|
324
|
+
return answer.serviceRevisions || [];
|
|
325
|
+
});
|
|
326
|
+
const taskDefinitionOf = new Map(
|
|
327
|
+
revisions.map((revision) => [
|
|
328
|
+
revision.serviceRevisionArn,
|
|
329
|
+
revision.taskDefinition
|
|
330
|
+
])
|
|
331
|
+
);
|
|
332
|
+
const matching = deployments.filter(
|
|
333
|
+
(deployment) =>
|
|
334
|
+
deployment.targetServiceRevision &&
|
|
335
|
+
taskDefinitionOf.get(deployment.targetServiceRevision.arn) ===
|
|
336
|
+
properties.taskDefinition
|
|
337
|
+
);
|
|
338
|
+
matching.sort(
|
|
339
|
+
(a, b) =>
|
|
340
|
+
(b.createdAt ? new Date(b.createdAt).getTime() : 0) -
|
|
341
|
+
(a.createdAt ? new Date(a.createdAt).getTime() : 0)
|
|
342
|
+
);
|
|
343
|
+
return matching[0];
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
/**
|
|
347
|
+
* `complete`, `waiting`, or `failed` with the reason, for the service's
|
|
348
|
+
* deployment of the task definition the barrier names.
|
|
349
|
+
*/
|
|
350
|
+
async function rolloutVerdict(
|
|
351
|
+
clients,
|
|
352
|
+
properties,
|
|
353
|
+
stackId,
|
|
354
|
+
waitStartedAt,
|
|
355
|
+
now,
|
|
356
|
+
abortSignal
|
|
357
|
+
) {
|
|
358
|
+
const live = await liveDeployments(clients.ecs, properties, abortSignal);
|
|
359
|
+
if (liveRolloutComplete(live, properties)) {
|
|
360
|
+
return { kind: "complete" };
|
|
361
|
+
}
|
|
362
|
+
if (await stackRollingBack(clients.stack, stackId, abortSignal)) {
|
|
363
|
+
return { kind: "complete", rollingBack: true };
|
|
364
|
+
}
|
|
365
|
+
const deployment = await newestDeploymentOf(
|
|
366
|
+
clients.ecs,
|
|
367
|
+
properties,
|
|
368
|
+
now,
|
|
369
|
+
abortSignal
|
|
370
|
+
);
|
|
371
|
+
if (deployment === undefined) {
|
|
372
|
+
if (atRestOnOrAfter(live, properties.taskDefinition)) {
|
|
373
|
+
return { kind: "complete", atRest: true };
|
|
374
|
+
}
|
|
375
|
+
return now - waitStartedAt >= NO_DEPLOYMENT_GRACE_MS
|
|
376
|
+
? {
|
|
377
|
+
kind: "failed",
|
|
378
|
+
reason:
|
|
379
|
+
`ECS lists no deployment of ${properties.taskDefinition} on ${properties.service} ` +
|
|
380
|
+
`from the last ${LISTING_WINDOW_MS / 3600000} hours, and the service did not come to rest ` +
|
|
381
|
+
`on it or a later revision of its family within ${NO_DEPLOYMENT_GRACE_MS / 60000} minutes`
|
|
382
|
+
}
|
|
383
|
+
: { kind: "waiting" };
|
|
384
|
+
}
|
|
385
|
+
if (SUCCEEDED.has(deployment.status)) return { kind: "complete" };
|
|
386
|
+
if (FAILED.has(deployment.status)) {
|
|
387
|
+
return {
|
|
388
|
+
kind: "failed",
|
|
389
|
+
reason:
|
|
390
|
+
`ECS ended the deployment of ${properties.taskDefinition} on ${properties.service} ` +
|
|
391
|
+
`${deployment.status}` +
|
|
392
|
+
(deployment.statusReason ? `: ${deployment.statusReason}` : "") +
|
|
393
|
+
` (${deployment.serviceDeploymentArn})`
|
|
394
|
+
};
|
|
395
|
+
}
|
|
396
|
+
return WAITING.has(deployment.status)
|
|
397
|
+
? { kind: "waiting" }
|
|
398
|
+
: { kind: "waiting", unknownStatus: String(deployment.status) };
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
async function isComplete(event, deps) {
|
|
402
|
+
if (event.RequestType === "Delete") return { IsComplete: true };
|
|
403
|
+
const properties = propertiesOf(event);
|
|
404
|
+
const stackId = event.StackId;
|
|
405
|
+
if (typeof stackId !== "string" || stackId === "") {
|
|
406
|
+
throw new Error("The rollout barrier's request names no stack");
|
|
407
|
+
}
|
|
408
|
+
const clients = {
|
|
409
|
+
ecs: (deps && deps.client) || getDefaultClient(),
|
|
410
|
+
stack: (deps && deps.stackClient) || getDefaultStackClient()
|
|
411
|
+
};
|
|
412
|
+
const now = (deps && deps.now) || Date.now();
|
|
413
|
+
const abortSignal =
|
|
414
|
+
(deps && deps.abortSignal) || AbortSignal.timeout(INVOCATION_BUDGET_MS);
|
|
415
|
+
const recorded = Number(event.Data && event.Data.WaitStartedAt);
|
|
416
|
+
const waitStartedAt =
|
|
417
|
+
Number.isFinite(recorded) && recorded > 0 ? recorded : now;
|
|
418
|
+
let verdict;
|
|
419
|
+
try {
|
|
420
|
+
verdict = await rolloutVerdict(
|
|
421
|
+
clients,
|
|
422
|
+
properties,
|
|
423
|
+
stackId,
|
|
424
|
+
waitStartedAt,
|
|
425
|
+
now,
|
|
426
|
+
abortSignal
|
|
427
|
+
);
|
|
428
|
+
} catch (err) {
|
|
429
|
+
const propagating =
|
|
430
|
+
isAccessDenied(err) && now - waitStartedAt < ACCESS_PROPAGATION_GRACE_MS;
|
|
431
|
+
if (!isTransient(err) && !propagating) throw err;
|
|
432
|
+
verdict = { kind: "waiting", transient: err.name || err.code || "unknown" };
|
|
433
|
+
}
|
|
434
|
+
console.log(
|
|
435
|
+
JSON.stringify({
|
|
436
|
+
msg: "rollout-barrier-verdict",
|
|
437
|
+
service: properties.service,
|
|
438
|
+
taskDefinition: properties.taskDefinition,
|
|
439
|
+
verdict: verdict.kind,
|
|
440
|
+
...(verdict.rollingBack ? { stackRollingBack: true } : {}),
|
|
441
|
+
...(verdict.atRest ? { atRest: true } : {}),
|
|
442
|
+
...(verdict.unknownStatus !== undefined
|
|
443
|
+
? { unknownStatus: verdict.unknownStatus }
|
|
444
|
+
: {}),
|
|
445
|
+
...(verdict.transient !== undefined
|
|
446
|
+
? { readFailed: verdict.transient }
|
|
447
|
+
: {})
|
|
448
|
+
})
|
|
449
|
+
);
|
|
450
|
+
if (verdict.kind === "failed") throw new Error(verdict.reason);
|
|
451
|
+
return { IsComplete: verdict.kind === "complete" };
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
exports.onEvent = onEvent;
|
|
455
|
+
exports.isComplete = isComplete;
|
|
456
|
+
|
|
457
|
+
exports._internals = {
|
|
458
|
+
propertiesOf,
|
|
459
|
+
isTransient,
|
|
460
|
+
isAccessDenied,
|
|
461
|
+
liveDeployments,
|
|
462
|
+
liveRolloutComplete,
|
|
463
|
+
atRestOnOrAfter,
|
|
464
|
+
familyRevisionOf,
|
|
465
|
+
stackRollingBack,
|
|
466
|
+
newestDeploymentOf,
|
|
467
|
+
rolloutVerdict,
|
|
468
|
+
SUCCEEDED,
|
|
469
|
+
FAILED,
|
|
470
|
+
WAITING,
|
|
471
|
+
NO_DEPLOYMENT_GRACE_MS,
|
|
472
|
+
ACCESS_PROPAGATION_GRACE_MS,
|
|
473
|
+
LISTING_WINDOW_MS,
|
|
474
|
+
INVOCATION_BUDGET_MS
|
|
475
|
+
};
|