canopycms-cdk 0.0.68-int.98 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -15
- package/dist/constructs/asset-support.d.ts +104 -84
- package/dist/constructs/asset-support.js +193 -195
- package/dist/constructs/cms-distribution.d.ts +2 -3
- package/dist/constructs/cms-distribution.js +5 -6
- package/dist/constructs/cms-service.d.ts +150 -18
- package/dist/constructs/cms-service.js +498 -94
- package/dist/constructs/editor-routing.d.ts +2 -2
- package/dist/constructs/worker-bundle.d.ts +42 -0
- package/dist/constructs/worker-bundle.js +118 -0
- package/dist/constructs/worker-lifecycle.d.ts +70 -0
- package/dist/constructs/worker-lifecycle.js +111 -0
- package/dist/constructs/worker-units.d.ts +99 -0
- package/dist/constructs/worker-units.js +398 -0
- package/dist/index.d.ts +2 -1
- package/lambda/asset-transform/dist/handler.js +813 -30360
- package/package.json +11 -6
- package/worker/canopy-github.service +81 -0
- package/worker/canopy-github.socket +16 -0
- package/worker/canopy-worker.service +76 -15
- package/worker/dist/index.js +77784 -40975
- package/worker/dist/index.js.contract +1 -0
- package/worker/dist/index.js.sha256 +1 -0
- package/worker/dist/instance-files.sh +365 -0
- package/worker/gateway.env.example +25 -0
|
@@ -1,16 +1,10 @@
|
|
|
1
|
-
import * as path from 'node:path';
|
|
2
|
-
import { fileURLToPath } from 'node:url';
|
|
3
1
|
import { Construct } from 'constructs';
|
|
4
|
-
import { Annotations, Duration, RemovalPolicy, Stack, Token, aws_ec2 as ec2, aws_efs as efs, aws_iam as iam, aws_lambda as lambda, aws_autoscaling as autoscaling,
|
|
2
|
+
import { Annotations, ArnFormat, Duration, Names, RemovalPolicy, Stack, Token, Tokenization, aws_cloudwatch as cloudwatch, aws_cloudwatch_actions as cloudwatchActions, aws_ec2 as ec2, aws_efs as efs, aws_iam as iam, aws_lambda as lambda, aws_autoscaling as autoscaling, aws_logs as logs, } from 'aws-cdk-lib';
|
|
5
3
|
import { attachLambdaExecutionPolicies } from './lambda-execution-role.js';
|
|
4
|
+
import { INSTANCE_FILES_DOWNLOAD_PATH, WORKER_BUNDLE_DOWNLOAD_PATH, workerBundleSource, } from './worker-bundle.js';
|
|
6
5
|
import { attachEditorBehaviors } from './editor-routing.js';
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
// below throws `__dirname is not defined` under a real ESM runtime (e.g. `tsx`)
|
|
10
|
-
// without this. Vitest's SSR/CJS-interop transform shims `__dirname`
|
|
11
|
-
// automatically, so this file's own tests would not catch its absence. Same fix
|
|
12
|
-
// as ../../lambda/asset-transform/build.mjs and ./asset-support.ts.
|
|
13
|
-
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
6
|
+
import { GATEWAY_SPLIT_CONTRACT, WORKER_CAPACITY_ENV, WORKER_CONTRACT_BANNER_PREFIX, WORKER_DRAIN_HOOK_NAME, WORKER_SYNC_LOG_PHRASE, } from './worker-lifecycle.js';
|
|
7
|
+
import { CANOPY_INSPECT, EFS_MOUNT_PATH, GATEWAY_LOG, GATEWAY_LOG_DIR, GATEWAY_USER, INSPECT_CHECKS, INSPECT_RC, INSTANCE_FILES_DIR, WORKER_INSTALL_DIR, WORKER_LOG, WORKER_LOG_DIR, } from './worker-units.js';
|
|
14
8
|
/**
|
|
15
9
|
* Synth-time mirror of `resolveDeploymentName`'s rule in the `canopycms`
|
|
16
10
|
* package (packages/canopycms/src/operating-mode/deployment-name.ts).
|
|
@@ -18,7 +12,7 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
|
18
12
|
* Duplicated rather than imported so a CONSTRUCT-only consumer does not pay for
|
|
19
13
|
* the core package's module graph. Whether that still justifies duplicating is
|
|
20
14
|
* an open question, tracked alongside the same question about the S3 prefix
|
|
21
|
-
* constants in
|
|
15
|
+
* constants in safeinsights/canopycms#608.
|
|
22
16
|
*
|
|
23
17
|
* Drift between the two copies is caught by a test, not by this comment:
|
|
24
18
|
* cms-deploy.test.ts drives both this construct and the runtime predicate over
|
|
@@ -36,21 +30,18 @@ const isValidDeploymentName = (name) => /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(name
|
|
|
36
30
|
!name.endsWith('.') &&
|
|
37
31
|
!name.endsWith('.lock');
|
|
38
32
|
/**
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
* One path string, not merely one directory: git records absolute paths on
|
|
43
|
-
* EFS (a workspace clone's `origin` is `<root>/remote.git`), and a path written
|
|
44
|
-
* by one process must resolve for the other.
|
|
33
|
+
* The text of the worker.log line user data writes when the boot's package
|
|
34
|
+
* upgrade failed and the worker runs on the AMI's packages. The unpatched
|
|
35
|
+
* alarm's metric filter counts it.
|
|
45
36
|
*/
|
|
46
|
-
const
|
|
37
|
+
const WORKER_UNPATCHED_LOG_PHRASE = 'running unpatched on the AMI packages until the next boot';
|
|
47
38
|
/**
|
|
48
|
-
* The heredoc delimiter user-data uses to write the worker's `.env`
|
|
49
|
-
* `cat > … << 'ENVEOF'`
|
|
39
|
+
* The heredoc delimiter user-data uses to write the worker's `.env` and the
|
|
40
|
+
* gateway's `gateway.env` (see the `cat > … << 'ENVEOF'` blocks below).
|
|
50
41
|
*/
|
|
51
42
|
const ENV_HEREDOC_DELIMITER = 'ENVEOF';
|
|
52
43
|
/**
|
|
53
|
-
* Guard every value interpolated into
|
|
44
|
+
* Guard every value interpolated into an env file (`.env` or `gateway.env`).
|
|
54
45
|
*
|
|
55
46
|
* This is robustness, not a security boundary: the heredoc delimiter is
|
|
56
47
|
* quoted, so the shell performs no expansion on the body, and these values
|
|
@@ -68,7 +59,7 @@ const ENV_HEREDOC_DELIMITER = 'ENVEOF';
|
|
|
68
59
|
function assertEnvSafe(name, value) {
|
|
69
60
|
if (/[\r\n]/.test(value)) {
|
|
70
61
|
throw new Error(`CanopyCmsService: ${name} must not contain a newline (got ${JSON.stringify(value)}). ` +
|
|
71
|
-
`It is written into
|
|
62
|
+
`It is written into an env file (.env or gateway.env), where a newline injects an arbitrary ` +
|
|
72
63
|
`extra environment line.`);
|
|
73
64
|
}
|
|
74
65
|
// systemd (EnvironmentFile=, see the unit below) treats a value whose FIRST
|
|
@@ -79,14 +70,14 @@ function assertEnvSafe(name, value) {
|
|
|
79
70
|
// which is why this checks position 0 rather than banning the character.
|
|
80
71
|
if (value.startsWith('"') || value.startsWith("'")) {
|
|
81
72
|
throw new Error(`CanopyCmsService: ${name} must not start with a quote character ` +
|
|
82
|
-
`(got ${JSON.stringify(value)}). It is written into
|
|
73
|
+
`(got ${JSON.stringify(value)}). It is written into an env file (.env or gateway.env), which ` +
|
|
83
74
|
`systemd reads as EnvironmentFile -- a leading quote opens a quoted value that ` +
|
|
84
75
|
`consumes every following line until a matching quote, silently emptying the rest ` +
|
|
85
|
-
`of the
|
|
76
|
+
`of the environment.`);
|
|
86
77
|
}
|
|
87
78
|
if (value.includes(ENV_HEREDOC_DELIMITER)) {
|
|
88
79
|
throw new Error(`CanopyCmsService: ${name} must not contain ${JSON.stringify(ENV_HEREDOC_DELIMITER)} ` +
|
|
89
|
-
`(got ${JSON.stringify(value)}). It is written into
|
|
80
|
+
`(got ${JSON.stringify(value)}). It is written into an env file (.env or gateway.env) with a ` +
|
|
90
81
|
`<< '${ENV_HEREDOC_DELIMITER}' heredoc, which that value would terminate early.`);
|
|
91
82
|
}
|
|
92
83
|
// The same systemd `EnvironmentFile=` parser the leading-quote rule above is
|
|
@@ -97,7 +88,7 @@ function assertEnvSafe(name, value) {
|
|
|
97
88
|
// in -- so it is refused here rather than debugged on an instance.
|
|
98
89
|
if (value.includes('\\')) {
|
|
99
90
|
throw new Error(`CanopyCmsService: ${name} must not contain a backslash (got ${JSON.stringify(value)}). ` +
|
|
100
|
-
`It is written into
|
|
91
|
+
`It is written into an env file (.env or gateway.env), which systemd reads as EnvironmentFile -- ` +
|
|
101
92
|
`there a backslash escapes the next character, and a trailing one continues the value ` +
|
|
102
93
|
`onto the following line, consuming the next variable entirely.`);
|
|
103
94
|
}
|
|
@@ -107,11 +98,74 @@ function assertEnvSafe(name, value) {
|
|
|
107
98
|
// by a route no charset check upstream can see.
|
|
108
99
|
if (value !== value.trim()) {
|
|
109
100
|
throw new Error(`CanopyCmsService: ${name} must not start or end with whitespace ` +
|
|
110
|
-
`(got ${JSON.stringify(value)}). systemd strips it when reading the
|
|
111
|
-
`the value the
|
|
101
|
+
`(got ${JSON.stringify(value)}). systemd strips it when reading the env file, so ` +
|
|
102
|
+
`the value the service sees would differ from the one configured here.`);
|
|
112
103
|
}
|
|
113
104
|
return value;
|
|
114
105
|
}
|
|
106
|
+
/**
|
|
107
|
+
* `lines` without their shell comment lines, outside heredoc bodies, so the comments stay in this
|
|
108
|
+
* file and out of the 16 KiB EC2 allows user data. A `#!` line is kept, and every line of a
|
|
109
|
+
* heredoc is kept as written. Only the quoted form, a line ending `<< 'DELIM'`, may open one: any
|
|
110
|
+
* other `<<` is refused, since its body would be read as commands and comments here.
|
|
111
|
+
* @internal Exported for tests.
|
|
112
|
+
*/
|
|
113
|
+
export function withoutShellComments(...lines) {
|
|
114
|
+
let heredocEnd;
|
|
115
|
+
return lines.filter((line) => {
|
|
116
|
+
if (heredocEnd !== undefined) {
|
|
117
|
+
if (line === heredocEnd)
|
|
118
|
+
heredocEnd = undefined;
|
|
119
|
+
return true;
|
|
120
|
+
}
|
|
121
|
+
if (/^\s*#(?!!)/.test(line))
|
|
122
|
+
return false;
|
|
123
|
+
if (line.includes('<<')) {
|
|
124
|
+
heredocEnd = /<< '(\w+)'$/.exec(line)?.[1];
|
|
125
|
+
if (heredocEnd === undefined)
|
|
126
|
+
throw new Error(`user data: unquoted heredoc in ${line}`);
|
|
127
|
+
}
|
|
128
|
+
return true;
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
/** EC2's limit on user data, in bytes, before base64. */
|
|
132
|
+
const USER_DATA_MAX_BYTES = 16_384;
|
|
133
|
+
/**
|
|
134
|
+
* The most bytes a deploy-time value the construct itself puts in user data resolves to: a bucket
|
|
135
|
+
* name is at most 63, an asset key or a sha256 under 80, a file system or access point id under 25.
|
|
136
|
+
*/
|
|
137
|
+
const USER_DATA_TOKEN_MAX_BYTES = 128;
|
|
138
|
+
/**
|
|
139
|
+
* The most bytes a deploy-time env value from a prop resolves to: a Secrets Manager ARN with a
|
|
140
|
+
* 512-character name is about 600. A longer token (an SSM parameter's value) is the adopter's to
|
|
141
|
+
* keep short.
|
|
142
|
+
*/
|
|
143
|
+
const ENV_TOKEN_MAX_BYTES = 640;
|
|
144
|
+
/** The most bytes a CloudWatch log group's name can be. */
|
|
145
|
+
const LOG_GROUP_NAME_MAX_BYTES = 512;
|
|
146
|
+
/**
|
|
147
|
+
* Add `lines` to `userData`, refusing at synth a configuration whose user data could exceed what
|
|
148
|
+
* EC2 accepts (EC2 would refuse the launch template, or every launch, and the group would never
|
|
149
|
+
* get a worker). Each token counts as its `bounds` entry's bytes, or
|
|
150
|
+
* {@link USER_DATA_TOKEN_MAX_BYTES}; an env value from a token longer than
|
|
151
|
+
* {@link ENV_TOKEN_MAX_BYTES} is not caught.
|
|
152
|
+
*/
|
|
153
|
+
function addUserData(userData, bounds, protectedBranchCount, ...lines) {
|
|
154
|
+
userData.addCommands(...lines);
|
|
155
|
+
const bytesOf = new Map(bounds.flatMap(({ token, bytes }) => Tokenization.reverseString(token).tokens.map((resolvable) => [resolvable, bytes])));
|
|
156
|
+
const longest = Tokenization.reverseString(userData.render())
|
|
157
|
+
.mapTokens({
|
|
158
|
+
mapToken: (token) => 'x'.repeat(bytesOf.get(token) ?? USER_DATA_TOKEN_MAX_BYTES),
|
|
159
|
+
})
|
|
160
|
+
.join({ join: (left, right) => `${String(left)}${String(right)}` });
|
|
161
|
+
const bytes = Buffer.byteLength(String(longest));
|
|
162
|
+
if (bytes > USER_DATA_MAX_BYTES) {
|
|
163
|
+
throw new Error(`CanopyCmsService: the worker's user data could reach ${bytes} bytes, over the ` +
|
|
164
|
+
`${USER_DATA_MAX_BYTES} EC2 accepts; protectedBranches has ${protectedBranchCount} ` +
|
|
165
|
+
`entries. Protect branches with fewer, wider * patterns, or shorten the other names ` +
|
|
166
|
+
`passed in (secret ARNs and fields, branches, the repository, the worker log group).`);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
115
169
|
/**
|
|
116
170
|
* Refuse a `baseBranch`/`settingsBranch` that git itself would reject, at
|
|
117
171
|
* synth rather than at worker boot.
|
|
@@ -130,9 +184,10 @@ function assertEnvSafe(name, value) {
|
|
|
130
184
|
* would refuse `cdk synth` for an adopter whose default branch is `release/v2`,
|
|
131
185
|
* and the worker handles such names fine - it keeps the raw name for git refs
|
|
132
186
|
* and runs it through `sanitizeBranchName` only for workspace DIRECTORY names.
|
|
133
|
-
* This implements git's `check-ref-format` rules for a branch name instead
|
|
134
|
-
*
|
|
135
|
-
*
|
|
187
|
+
* This implements git's `check-ref-format` rules for a branch name instead,
|
|
188
|
+
* plus the GitHub gateway's 255-character cap: the gateway refuses a longer
|
|
189
|
+
* pull request base. A settings branch is pushed through the gateway, so it
|
|
190
|
+
* takes the gateway's narrower name rules too (`assertGatewayBranchName`).
|
|
136
191
|
*/
|
|
137
192
|
function assertValidGitBranchName(propName, value) {
|
|
138
193
|
const reject = (why) => {
|
|
@@ -145,6 +200,8 @@ function assertValidGitBranchName(propName, value) {
|
|
|
145
200
|
};
|
|
146
201
|
if (value.length === 0)
|
|
147
202
|
reject('it is empty');
|
|
203
|
+
if (value.length > 255)
|
|
204
|
+
reject('it is longer than 255 characters');
|
|
148
205
|
if (value === '@')
|
|
149
206
|
reject("a lone '@' is reserved by git");
|
|
150
207
|
// `git check-ref-format --branch` rejects HEAD: it names the symbolic ref,
|
|
@@ -182,6 +239,67 @@ function assertValidGitBranchName(propName, value) {
|
|
|
182
239
|
}
|
|
183
240
|
return value;
|
|
184
241
|
}
|
|
242
|
+
/**
|
|
243
|
+
* Refuse a name the GitHub gateway would refuse to push (`branchNameProblem`,
|
|
244
|
+
* packages/canopycms/src/worker/branch-policy.ts), on top of git's rules: printable ASCII, no
|
|
245
|
+
* `"`, `%`, `{` or `}`, no leading `refs/`, `heads/`, `tags/` or `remotes/`.
|
|
246
|
+
*/
|
|
247
|
+
function assertGatewayBranchName(propName, value) {
|
|
248
|
+
const reject = (why) => {
|
|
249
|
+
throw new Error(`CanopyCmsService: invalid ${propName} ${JSON.stringify(value)} -- ${why}.`);
|
|
250
|
+
};
|
|
251
|
+
if (!/^[\x20-\x7e]*$/.test(value))
|
|
252
|
+
reject('it is not printable ASCII');
|
|
253
|
+
if (/["%{}]/.test(value))
|
|
254
|
+
reject('it contains ", %, { or }');
|
|
255
|
+
if (/^(?:refs|heads|tags|remotes)\//i.test(value)) {
|
|
256
|
+
reject("it starts with 'refs/', 'heads/', 'tags/' or 'remotes/'");
|
|
257
|
+
}
|
|
258
|
+
return value;
|
|
259
|
+
}
|
|
260
|
+
/**
|
|
261
|
+
* The most `protectedBranches` entries the gateway takes
|
|
262
|
+
* (`MAX_PROTECTED_BRANCHES`, packages/canopycms/src/worker/branch-policy.ts). A local constant:
|
|
263
|
+
* this module imports nothing from core.
|
|
264
|
+
*/
|
|
265
|
+
const MAX_PROTECTED_BRANCHES = 1024;
|
|
266
|
+
/**
|
|
267
|
+
* Refuse `protectedBranches` the GitHub gateway would refuse at boot, which would crash-loop it
|
|
268
|
+
* (`protectedPatternProblem`, packages/canopycms/src/worker/branch-policy.ts): more than
|
|
269
|
+
* {@link MAX_PROTECTED_BRANCHES} entries, and any entry that is not a branch name in which `*` is
|
|
270
|
+
* a wildcard, held to git's rules and the gateway's (`assertGatewayBranchName`). No entry may
|
|
271
|
+
* start with the reserved settings prefix: a glob there matches nothing, and a name there would
|
|
272
|
+
* stop the deployment's settings reaching GitHub, since a settings branch is pushed plainly every
|
|
273
|
+
* sync. A comma separates entries in the env file, so none may hold one.
|
|
274
|
+
*/
|
|
275
|
+
function assertValidProtectedBranches(entries) {
|
|
276
|
+
if (entries.length > MAX_PROTECTED_BRANCHES) {
|
|
277
|
+
throw new Error(`CanopyCmsService: protectedBranches has ${entries.length} entries; the most it takes is ${MAX_PROTECTED_BRANCHES}.`);
|
|
278
|
+
}
|
|
279
|
+
return entries.map((entry) => {
|
|
280
|
+
const reject = (why) => {
|
|
281
|
+
throw new Error(`CanopyCmsService: invalid protectedBranches entry ${JSON.stringify(entry)} -- ${why}.`);
|
|
282
|
+
};
|
|
283
|
+
if (entry.includes(','))
|
|
284
|
+
reject('it contains a comma, which separates entries');
|
|
285
|
+
assertGatewayBranchName('protectedBranches entry', entry);
|
|
286
|
+
if (entry.startsWith('canopycms-settings-')) {
|
|
287
|
+
reject(entry.includes('*')
|
|
288
|
+
? 'the worker never lets a glob match a settings branch, so this one would match nothing'
|
|
289
|
+
: "a settings branch is pushed plainly every sync, so protecting it would stop this deployment's settings reaching GitHub");
|
|
290
|
+
}
|
|
291
|
+
const asName = entry.replace(/\*/g, 'x');
|
|
292
|
+
try {
|
|
293
|
+
assertValidGitBranchName('protectedBranches entry', asName);
|
|
294
|
+
}
|
|
295
|
+
catch (err) {
|
|
296
|
+
// Name the entry as written, not with its wildcards replaced.
|
|
297
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
298
|
+
throw new Error(message.replace(JSON.stringify(asName), JSON.stringify(entry)));
|
|
299
|
+
}
|
|
300
|
+
return entry;
|
|
301
|
+
});
|
|
302
|
+
}
|
|
185
303
|
/**
|
|
186
304
|
* A Secrets Manager ARN carrying the ECS/CloudFormation JSON-field suffix,
|
|
187
305
|
* i.e. `arn:…:secret:name-AbCdEf:MY_KEY::` rather than `arn:…:secret:name-AbCdEf`.
|
|
@@ -317,7 +435,7 @@ const GITHUB_TOKEN_PROP_NAMES = ['githubTokenSecretArn', 'githubTokenSecretJsonF
|
|
|
317
435
|
* Rejects a PEM private key passed where a prop expects an identifier or an ARN.
|
|
318
436
|
*
|
|
319
437
|
* There is no plaintext private-key prop, and there cannot be one: the value
|
|
320
|
-
* would be written into
|
|
438
|
+
* would be written into `gateway.env`, which systemd reads as
|
|
321
439
|
* `EnvironmentFile=` where a newline begins a new variable. So the realistic
|
|
322
440
|
* mistake is to paste the key into `githubAppPrivateKeySecretArn` -- the prop
|
|
323
441
|
* whose name contains "PrivateKey" -- instead of the ARN of a secret holding it.
|
|
@@ -346,7 +464,7 @@ function assertNotInlinePrivateKey(propName, value) {
|
|
|
346
464
|
return;
|
|
347
465
|
throw new Error(`CanopyCmsService: ${propName} looks like a PEM private key, not ${propName === 'githubAppPrivateKeySecretArn' ? 'a secret ARN' : 'an identifier'}. ` +
|
|
348
466
|
`The GitHub App private key can ONLY be supplied as a Secrets Manager ARN -- it is ` +
|
|
349
|
-
`multi-line, and every value this construct configures goes into
|
|
467
|
+
`multi-line, and every value this construct configures goes into an env file (.env or gateway.env), ` +
|
|
350
468
|
`which systemd reads as EnvironmentFile where a newline starts a new variable. Store the ` +
|
|
351
469
|
`PEM in Secrets Manager and pass that secret's full ARN as githubAppPrivateKeySecretArn ` +
|
|
352
470
|
`(optionally with githubAppPrivateKeySecretJsonField if it lives inside a JSON document).`);
|
|
@@ -430,7 +548,7 @@ function assertGitHubAuthProps(props) {
|
|
|
430
548
|
`drop them and use githubTokenSecretArn.`);
|
|
431
549
|
}
|
|
432
550
|
// The JSON field counts as "a token is configured", not just the ARN. An
|
|
433
|
-
// adopter following
|
|
551
|
+
// adopter following CHANGELOG.md removes `githubTokenSecretArn`
|
|
434
552
|
// and overlooks `githubTokenSecretJsonField`; left out of this check, that
|
|
435
553
|
// lands on `assertSecretPropPair` instead, which answers "Set
|
|
436
554
|
// githubTokenSecretArn, or drop githubTokenSecretJsonField" -- pointing them
|
|
@@ -474,6 +592,31 @@ export const DEFAULT_CMS_LAMBDA_TIMEOUT = Duration.seconds(60);
|
|
|
474
592
|
export const MIN_CMS_RESERVED_CONCURRENCY = 20;
|
|
475
593
|
/** Default `reservedConcurrency`; see that prop for what it caps and costs. */
|
|
476
594
|
export const DEFAULT_CMS_RESERVED_CONCURRENCY = 50;
|
|
595
|
+
const DEFAULT_SPOT_INSTANCE_TYPES = ['t4g.nano', 't4g.micro', 't4g.small'];
|
|
596
|
+
/** The worker's internal drain deadline (core's DEFAULT_DRAIN_DEADLINE_MS) plus two minutes. */
|
|
597
|
+
export const MIN_WORKER_TERMINATION_HEARTBEAT = Duration.seconds(210);
|
|
598
|
+
const MAX_WORKER_TERMINATION_HEARTBEAT = Duration.hours(2);
|
|
599
|
+
/** Graviton families: `a1`, or a generation digit followed by `g` (t4g, c7gn, m8gd). */
|
|
600
|
+
function assertGravitonInstanceType(instanceType) {
|
|
601
|
+
const name = instanceType.toString();
|
|
602
|
+
if (Token.isUnresolved(name))
|
|
603
|
+
return;
|
|
604
|
+
if (!/^(a1|[a-z]+\d+g[a-z]*)\./.test(name)) {
|
|
605
|
+
throw new Error(`workerCapacity: instance type "${name}" is not Graviton (arm64). The worker's AMI is ` +
|
|
606
|
+
'Amazon Linux 2023 for arm64, so an x86 instance cannot boot it.');
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
function assertWorkerTerminationHeartbeat(heartbeat) {
|
|
610
|
+
if (Token.isUnresolved(heartbeat.toSeconds({ integral: false })))
|
|
611
|
+
return;
|
|
612
|
+
const seconds = heartbeat.toSeconds();
|
|
613
|
+
if (seconds < MIN_WORKER_TERMINATION_HEARTBEAT.toSeconds() ||
|
|
614
|
+
seconds > MAX_WORKER_TERMINATION_HEARTBEAT.toSeconds()) {
|
|
615
|
+
throw new Error(`workerTerminationHeartbeat must be between ${MIN_WORKER_TERMINATION_HEARTBEAT.toSeconds()} ` +
|
|
616
|
+
`and ${MAX_WORKER_TERMINATION_HEARTBEAT.toSeconds()} seconds (got ${seconds}): shorter ` +
|
|
617
|
+
"cuts off the worker's 90-second drain, and Auto Scaling allows at most two hours.");
|
|
618
|
+
}
|
|
619
|
+
}
|
|
477
620
|
/**
|
|
478
621
|
* Core CDK construct for CanopyCMS deployment.
|
|
479
622
|
*
|
|
@@ -482,7 +625,8 @@ export const DEFAULT_CMS_RESERVED_CONCURRENCY = 50;
|
|
|
482
625
|
* - EFS filesystem with access point at /workspace
|
|
483
626
|
* - Lambda function (Docker image, EFS mount, private subnet, no internet)
|
|
484
627
|
* - Lambda Function URL (for CloudFront origin)
|
|
485
|
-
* - EC2 Worker (t4g.nano
|
|
628
|
+
* - EC2 Worker (on-demand t4g.nano by default, in an ASG, public subnet, EFS
|
|
629
|
+
* mount, systemd, a terminating lifecycle hook it completes once drained) -
|
|
486
630
|
* rolled via the ASG's UpdatePolicy by every deploy that changes its launch
|
|
487
631
|
* template, so a changed worker bundle reaches the instance instead of
|
|
488
632
|
* sitting unused in a launch template until the next spot interruption (see
|
|
@@ -526,19 +670,43 @@ export class CanopyCmsService extends Construct {
|
|
|
526
670
|
}
|
|
527
671
|
// Both are interpolated into a git ref and the worker's `.env` heredoc, so
|
|
528
672
|
// both are guarded at synth - see their doc comments for the failure each
|
|
529
|
-
// prevents.
|
|
530
|
-
//
|
|
531
|
-
//
|
|
673
|
+
// prevents. Each stays `undefined` (not stamped at all) unless the adopter
|
|
674
|
+
// explicitly set it, so the worker detects the base branch and computes the
|
|
675
|
+
// settings branch name. For the settings branch an absent env var and an
|
|
676
|
+
// empty one are NOT the same: the worker computes the name only when
|
|
532
677
|
// `CANOPYCMS_SETTINGS_BRANCH` is unset entirely.
|
|
533
|
-
const baseBranch =
|
|
678
|
+
const baseBranch = props.baseBranch !== undefined
|
|
679
|
+
? assertValidGitBranchName('baseBranch', props.baseBranch)
|
|
680
|
+
: undefined;
|
|
534
681
|
const settingsBranch = props.settingsBranch !== undefined
|
|
535
682
|
? assertValidGitBranchName('settingsBranch', props.settingsBranch)
|
|
536
683
|
: undefined;
|
|
684
|
+
const protectedBranches = assertValidProtectedBranches(props.protectedBranches ?? []);
|
|
685
|
+
const workerCapacity = props.workerCapacity ?? { type: 'on-demand' };
|
|
686
|
+
const workerInstanceTypes = workerCapacity.type === 'spot'
|
|
687
|
+
? (workerCapacity.instanceTypes ??
|
|
688
|
+
DEFAULT_SPOT_INSTANCE_TYPES.map((name) => new ec2.InstanceType(name)))
|
|
689
|
+
: [
|
|
690
|
+
workerCapacity.instanceType ??
|
|
691
|
+
ec2.InstanceType.of(ec2.InstanceClass.T4G, ec2.InstanceSize.NANO),
|
|
692
|
+
];
|
|
693
|
+
if (workerInstanceTypes.length === 0) {
|
|
694
|
+
throw new Error('workerCapacity.instanceTypes must name at least one instance type');
|
|
695
|
+
}
|
|
696
|
+
workerInstanceTypes.forEach(assertGravitonInstanceType);
|
|
697
|
+
if (workerCapacity.type === 'spot' && workerCapacity.maxPrice !== undefined) {
|
|
698
|
+
const price = Number(workerCapacity.maxPrice);
|
|
699
|
+
if (!Number.isFinite(price) || price <= 0) {
|
|
700
|
+
throw new Error(`workerCapacity.maxPrice must be a positive USD-per-hour amount (got "${workerCapacity.maxPrice}")`);
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
const workerTerminationHeartbeat = props.workerTerminationHeartbeat ?? Duration.minutes(5);
|
|
704
|
+
assertWorkerTerminationHeartbeat(workerTerminationHeartbeat);
|
|
537
705
|
// Checked at the top so a misconfigured pair fails `cdk synth` rather than
|
|
538
706
|
// `cdk deploy`-then-restart-loop.
|
|
539
707
|
//
|
|
540
708
|
// WHICH CREDENTIAL first, then whether each is well formed: the order is
|
|
541
|
-
// load-bearing. Reversed, an adopter following
|
|
709
|
+
// load-bearing. Reversed, an adopter following CHANGELOG.md who
|
|
542
710
|
// removes `githubTokenSecretArn` and overlooks `githubTokenSecretJsonField`
|
|
543
711
|
// is told "Set githubTokenSecretArn, or drop githubTokenSecretJsonField" --
|
|
544
712
|
// pointing them back at the credential they were just told to delete, and at
|
|
@@ -621,6 +789,19 @@ export class CanopyCmsService extends Construct {
|
|
|
621
789
|
description: 'CanopyCMS EFS',
|
|
622
790
|
allowAllOutbound: false,
|
|
623
791
|
});
|
|
792
|
+
// With any policy in effect, EFS evaluates a client that mounts without
|
|
793
|
+
// `iam` against the Principal "*" statements alone, so a policy granting
|
|
794
|
+
// "*" nothing is what requires IAM: the worker and the Lambda get their
|
|
795
|
+
// access from their roles' grants (below, and CDK's for the Lambda's access
|
|
796
|
+
// point). Lambda uses TLS for every file-system connection
|
|
797
|
+
// (docs.aws.amazon.com/lambda/latest/dg/security-dataprotection.html). `allowAnonymousAccess: true` keeps CDK from adding its own
|
|
798
|
+
// "*" ClientWrite/ClientRootAccess statement, which it does under the
|
|
799
|
+
// `@aws-cdk/aws-efs:denyAnonymousAccess` flag or after a `grant*` call.
|
|
800
|
+
const efsClientActions = [
|
|
801
|
+
'elasticfilesystem:ClientMount',
|
|
802
|
+
'elasticfilesystem:ClientWrite',
|
|
803
|
+
'elasticfilesystem:ClientRootAccess',
|
|
804
|
+
];
|
|
624
805
|
this.fileSystem = new efs.FileSystem(this, 'FileSystem', {
|
|
625
806
|
vpc: this.vpc,
|
|
626
807
|
encrypted: true,
|
|
@@ -628,6 +809,20 @@ export class CanopyCmsService extends Construct {
|
|
|
628
809
|
removalPolicy: props.efsRemovalPolicy ?? RemovalPolicy.RETAIN,
|
|
629
810
|
vpcSubnets: { subnetType: ec2.SubnetType.PRIVATE_ISOLATED },
|
|
630
811
|
securityGroup: efsSg,
|
|
812
|
+
enableAutomaticBackups: props.efsBackup ?? true,
|
|
813
|
+
allowAnonymousAccess: true,
|
|
814
|
+
fileSystemPolicy: (props.efsEnforceIamAndTls ?? true)
|
|
815
|
+
? new iam.PolicyDocument({
|
|
816
|
+
statements: [
|
|
817
|
+
new iam.PolicyStatement({
|
|
818
|
+
effect: iam.Effect.DENY,
|
|
819
|
+
principals: [new iam.AnyPrincipal()],
|
|
820
|
+
actions: efsClientActions,
|
|
821
|
+
conditions: { Bool: { 'aws:SecureTransport': 'false' } },
|
|
822
|
+
}),
|
|
823
|
+
],
|
|
824
|
+
})
|
|
825
|
+
: undefined,
|
|
631
826
|
});
|
|
632
827
|
const accessPoint = this.fileSystem.addAccessPoint('WorkspaceAP', {
|
|
633
828
|
path: '/workspace',
|
|
@@ -823,8 +1018,26 @@ export class CanopyCmsService extends Construct {
|
|
|
823
1018
|
resources: secretsArns,
|
|
824
1019
|
}));
|
|
825
1020
|
}
|
|
826
|
-
//
|
|
827
|
-
|
|
1021
|
+
// The worker's `iam` mount, through the shared access point only. The
|
|
1022
|
+
// describe calls serve amazon-efs-utils' fallback to a mount target's IP
|
|
1023
|
+
// address when the file system's DNS name does not resolve (it needs
|
|
1024
|
+
// botocore too).
|
|
1025
|
+
workerRole.addToPolicy(new iam.PolicyStatement({
|
|
1026
|
+
actions: ['elasticfilesystem:ClientMount', 'elasticfilesystem:ClientWrite'],
|
|
1027
|
+
resources: [this.fileSystem.fileSystemArn],
|
|
1028
|
+
conditions: {
|
|
1029
|
+
StringEquals: { 'elasticfilesystem:AccessPointArn': accessPoint.accessPointArn },
|
|
1030
|
+
},
|
|
1031
|
+
}));
|
|
1032
|
+
workerRole.addToPolicy(new iam.PolicyStatement({
|
|
1033
|
+
actions: ['elasticfilesystem:DescribeMountTargets'],
|
|
1034
|
+
resources: [this.fileSystem.fileSystemArn],
|
|
1035
|
+
}));
|
|
1036
|
+
workerRole.addToPolicy(new iam.PolicyStatement({
|
|
1037
|
+
// Supports no resource-level permissions.
|
|
1038
|
+
actions: ['ec2:DescribeAvailabilityZones'],
|
|
1039
|
+
resources: ['*'],
|
|
1040
|
+
}));
|
|
828
1041
|
// Observation channel for a NAT-less deploy (SSM Session Manager / send-command);
|
|
829
1042
|
// worker SG already allows 443 egress.
|
|
830
1043
|
workerRole.addManagedPolicy(iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore'));
|
|
@@ -839,41 +1052,110 @@ export class CanopyCmsService extends Construct {
|
|
|
839
1052
|
// CreateLogStream + PutLogEvents scoped to this group only (least privilege;
|
|
840
1053
|
// the group is pre-created by CFN so the agent never needs CreateLogGroup).
|
|
841
1054
|
this.workerLogGroup.grantWrite(workerRole);
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
1055
|
+
if (props.alarmTopic) {
|
|
1056
|
+
// Metrics are keyed by namespace and name alone, so the name carries this
|
|
1057
|
+
// construct's unique id: two services in one account and region must not
|
|
1058
|
+
// share one count. Capped at CloudWatch's 255 characters.
|
|
1059
|
+
const metricPrefix = 'WorkerGitSyncCycles-';
|
|
1060
|
+
const syncCycles = new logs.MetricFilter(this, 'WorkerSyncCycles', {
|
|
1061
|
+
logGroup: this.workerLogGroup,
|
|
1062
|
+
filterPattern: logs.FilterPattern.literal(`"${WORKER_SYNC_LOG_PHRASE}"`),
|
|
1063
|
+
metricNamespace: 'CanopyCMS',
|
|
1064
|
+
metricName: `${metricPrefix}${Names.uniqueResourceName(this, { maxLength: 255 - metricPrefix.length })}`,
|
|
1065
|
+
metricValue: '1',
|
|
1066
|
+
});
|
|
1067
|
+
// Fires when 3 consecutive 10-minute periods hold no git-sync cycle line.
|
|
1068
|
+
// The worker logs one every cycle (default 5 minutes), even while the
|
|
1069
|
+
// schema gate holds the base branch. 30 minutes leaves room for a normal
|
|
1070
|
+
// replacement: the terminating hook's drain
|
|
1071
|
+
// (`workerTerminationHeartbeat`, 5 minutes by default), a boot of a few
|
|
1072
|
+
// minutes plus its package upgrade, and the first sync. So a deploy or a
|
|
1073
|
+
// spot replacement does not page, while a crash loop, a boot loop, a worker that never started and a
|
|
1074
|
+
// worker whose loops have stopped all go silent and do. Missing data
|
|
1075
|
+
// counts as a breach because a boot loop dies before the CloudWatch agent
|
|
1076
|
+
// is installed and so writes no events at all. The construct does not
|
|
1077
|
+
// expose CANOPYCMS_GIT_SYNC_INTERVAL; an adopter who sets it above about
|
|
1078
|
+
// 10 minutes on the instance needs a wider window.
|
|
1079
|
+
this.workerDownAlarm = new cloudwatch.Alarm(this, 'WorkerDownAlarm', {
|
|
1080
|
+
metric: syncCycles.metric({ statistic: 'Sum', period: Duration.minutes(10) }),
|
|
1081
|
+
threshold: 1,
|
|
1082
|
+
comparisonOperator: cloudwatch.ComparisonOperator.LESS_THAN_THRESHOLD,
|
|
1083
|
+
evaluationPeriods: 3,
|
|
1084
|
+
datapointsToAlarm: 3,
|
|
1085
|
+
treatMissingData: cloudwatch.TreatMissingData.BREACHING,
|
|
1086
|
+
alarmDescription: `The CMS worker has not run a git sync for 30 minutes, so publishing and branch syncing ` +
|
|
1087
|
+
`are stalled. Check System health in the editor's admin panel and the worker log group ` +
|
|
1088
|
+
`${this.workerLogGroup.logGroupName}.`,
|
|
1089
|
+
});
|
|
1090
|
+
const topicAction = new cloudwatchActions.SnsAction(props.alarmTopic);
|
|
1091
|
+
this.workerDownAlarm.addAlarmAction(topicAction);
|
|
1092
|
+
this.workerDownAlarm.addOkAction(topicAction);
|
|
1093
|
+
const unpatchedPrefix = 'WorkerUnpatchedBoots-';
|
|
1094
|
+
const unpatchedBoots = new logs.MetricFilter(this, 'WorkerUnpatchedBoots', {
|
|
1095
|
+
logGroup: this.workerLogGroup,
|
|
1096
|
+
filterPattern: logs.FilterPattern.literal(`"${WORKER_UNPATCHED_LOG_PHRASE}"`),
|
|
1097
|
+
metricNamespace: 'CanopyCMS',
|
|
1098
|
+
metricName: `${unpatchedPrefix}${Names.uniqueResourceName(this, { maxLength: 255 - unpatchedPrefix.length })}`,
|
|
1099
|
+
metricValue: '1',
|
|
1100
|
+
});
|
|
1101
|
+
// One line per unpatched boot, so the alarm returns to OK one period
|
|
1102
|
+
// later on its own; an OK notification would only repeat the alarm.
|
|
1103
|
+
this.workerUnpatchedAlarm = new cloudwatch.Alarm(this, 'WorkerUnpatchedAlarm', {
|
|
1104
|
+
metric: unpatchedBoots.metric({ statistic: 'Sum', period: Duration.minutes(10) }),
|
|
1105
|
+
threshold: 1,
|
|
1106
|
+
comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
|
|
1107
|
+
evaluationPeriods: 1,
|
|
1108
|
+
treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
|
|
1109
|
+
alarmDescription: `A CMS worker boot could not upgrade its packages and started on the AMI's. It is ` +
|
|
1110
|
+
`working, and the next boot retries the upgrade. The boot's console output ` +
|
|
1111
|
+
`(aws ec2 get-console-output) and the worker log group ` +
|
|
1112
|
+
`${this.workerLogGroup.logGroupName} name the failure.`,
|
|
1113
|
+
});
|
|
1114
|
+
this.workerUnpatchedAlarm.addAlarmAction(topicAction);
|
|
1115
|
+
}
|
|
1116
|
+
const workerBundle = workerBundleSource(this, props.workerCode ?? { source: 'asset' }, workerRole);
|
|
1117
|
+
this.workerBundleBucket = workerBundle.bundleBucket;
|
|
1118
|
+
this.workerBundleSha256Parameter = workerBundle.sha256Parameter;
|
|
847
1119
|
// Name/value PAIRS rather than pre-formatted lines: every value then flows
|
|
848
1120
|
// through `assertEnvSafe` in the single `map` below, so a value added here
|
|
849
1121
|
// later is guarded whether or not whoever adds it remembers to.
|
|
1122
|
+
//
|
|
1123
|
+
// Each also names the process that reads it under the split. The worker
|
|
1124
|
+
// gets only what it needs to find the repository and the workspace; every
|
|
1125
|
+
// secret, every name of one (ARNs, JSON fields, the App's ids), what
|
|
1126
|
+
// reaches AWS, and what only the gateway enforces go to gateway.env, which
|
|
1127
|
+
// the worker cannot read. The single-process unit reads both files.
|
|
850
1128
|
const envEntries = [
|
|
851
|
-
['CANOPYCMS_WORKSPACE_ROOT', EFS_MOUNT_PATH],
|
|
852
|
-
['CANOPYCMS_GITHUB_OWNER', props.githubOwner],
|
|
853
|
-
['CANOPYCMS_GITHUB_REPO', props.githubRepo],
|
|
854
|
-
['CANOPYCMS_BASE_BRANCH', baseBranch],
|
|
1129
|
+
['CANOPYCMS_WORKSPACE_ROOT', EFS_MOUNT_PATH, 'worker'],
|
|
1130
|
+
['CANOPYCMS_GITHUB_OWNER', props.githubOwner, 'both'],
|
|
1131
|
+
['CANOPYCMS_GITHUB_REPO', props.githubRepo, 'both'],
|
|
855
1132
|
// The SAME string the Lambda's environment gets above, including an
|
|
856
1133
|
// `environment.CANOPYCMS_DEPLOYMENT_NAME` override - the two halves
|
|
857
1134
|
// resolve one settings branch (`canopycms-settings-<name>`) between them,
|
|
858
1135
|
// and disagreeing here is what the runtime warning detects.
|
|
859
|
-
['CANOPYCMS_DEPLOYMENT_NAME', deploymentName],
|
|
1136
|
+
['CANOPYCMS_DEPLOYMENT_NAME', deploymentName, 'worker'],
|
|
860
1137
|
// The AWS SDK JS v3 cannot resolve a region from IMDS on its own - without
|
|
861
|
-
// this
|
|
862
|
-
//
|
|
863
|
-
['AWS_REGION', Stack.of(this).region],
|
|
1138
|
+
// this a bare `SecretsManagerClient({})` crash-loops with "Region is
|
|
1139
|
+
// missing".
|
|
1140
|
+
['AWS_REGION', Stack.of(this).region, 'gateway'],
|
|
864
1141
|
];
|
|
1142
|
+
if (baseBranch !== undefined) {
|
|
1143
|
+
// The worker syncs it; the gateway never pushes it.
|
|
1144
|
+
envEntries.push(['CANOPYCMS_BASE_BRANCH', baseBranch, 'both']);
|
|
1145
|
+
}
|
|
865
1146
|
if (props.githubTokenSecretArn) {
|
|
866
|
-
envEntries.push(['CANOPYCMS_GITHUB_TOKEN_SECRET_ARN', props.githubTokenSecretArn]);
|
|
1147
|
+
envEntries.push(['CANOPYCMS_GITHUB_TOKEN_SECRET_ARN', props.githubTokenSecretArn, 'gateway']);
|
|
867
1148
|
}
|
|
868
1149
|
// The JSON-field vars need NO IAM change, and that is not an oversight: a
|
|
869
1150
|
// field is a key inside a secret's value, not a separately grantable
|
|
870
1151
|
// resource. `secretsmanager:GetSecretValue` on the secret -- already granted
|
|
871
|
-
// above from the same ARN prop -- returns the whole document, and the
|
|
1152
|
+
// above from the same ARN prop -- returns the whole document, and the gateway
|
|
872
1153
|
// picks the field out of it in `getSecret` (worker/secrets.ts).
|
|
873
1154
|
if (props.githubTokenSecretJsonField) {
|
|
874
1155
|
envEntries.push([
|
|
875
1156
|
'CANOPYCMS_GITHUB_TOKEN_SECRET_JSON_FIELD',
|
|
876
1157
|
props.githubTokenSecretJsonField,
|
|
1158
|
+
'gateway',
|
|
877
1159
|
]);
|
|
878
1160
|
}
|
|
879
1161
|
// GitHub App credentials, stamped individually even though
|
|
@@ -881,56 +1163,102 @@ export class CanopyCmsService extends Construct {
|
|
|
881
1163
|
// future prop added to the App set then cannot be silently dropped by a
|
|
882
1164
|
// condition that names only its siblings.
|
|
883
1165
|
if (props.githubAppId) {
|
|
884
|
-
envEntries.push(['CANOPYCMS_GITHUB_APP_ID', props.githubAppId]);
|
|
1166
|
+
envEntries.push(['CANOPYCMS_GITHUB_APP_ID', props.githubAppId, 'gateway']);
|
|
885
1167
|
}
|
|
886
1168
|
if (props.githubAppInstallationId) {
|
|
887
|
-
envEntries.push([
|
|
1169
|
+
envEntries.push([
|
|
1170
|
+
'CANOPYCMS_GITHUB_APP_INSTALLATION_ID',
|
|
1171
|
+
props.githubAppInstallationId,
|
|
1172
|
+
'gateway',
|
|
1173
|
+
]);
|
|
888
1174
|
}
|
|
889
1175
|
if (props.githubAppPrivateKeySecretArn) {
|
|
890
1176
|
envEntries.push([
|
|
891
1177
|
'CANOPYCMS_GITHUB_APP_PRIVATE_KEY_SECRET_ARN',
|
|
892
1178
|
props.githubAppPrivateKeySecretArn,
|
|
1179
|
+
'gateway',
|
|
893
1180
|
]);
|
|
894
1181
|
}
|
|
895
1182
|
if (props.githubAppPrivateKeySecretJsonField) {
|
|
896
1183
|
envEntries.push([
|
|
897
1184
|
'CANOPYCMS_GITHUB_APP_PRIVATE_KEY_SECRET_JSON_FIELD',
|
|
898
1185
|
props.githubAppPrivateKeySecretJsonField,
|
|
1186
|
+
'gateway',
|
|
899
1187
|
]);
|
|
900
1188
|
}
|
|
901
1189
|
if (props.clerkSecretKeySecretArn) {
|
|
902
|
-
envEntries.push(['CLERK_SECRET_KEY_SECRET_ARN', props.clerkSecretKeySecretArn]);
|
|
1190
|
+
envEntries.push(['CLERK_SECRET_KEY_SECRET_ARN', props.clerkSecretKeySecretArn, 'gateway']);
|
|
903
1191
|
}
|
|
904
1192
|
if (props.clerkSecretKeySecretJsonField) {
|
|
905
|
-
envEntries.push([
|
|
1193
|
+
envEntries.push([
|
|
1194
|
+
'CLERK_SECRET_KEY_SECRET_JSON_FIELD',
|
|
1195
|
+
props.clerkSecretKeySecretJsonField,
|
|
1196
|
+
'gateway',
|
|
1197
|
+
]);
|
|
1198
|
+
}
|
|
1199
|
+
if (workerCapacity.type === 'spot') {
|
|
1200
|
+
// The termination watch is the gateway's.
|
|
1201
|
+
envEntries.push([WORKER_CAPACITY_ENV, 'spot', 'gateway']);
|
|
906
1202
|
}
|
|
907
1203
|
if (settingsBranch !== undefined) {
|
|
908
1204
|
// Only when explicitly set - an absent prop must keep today's behavior
|
|
909
1205
|
// (the worker falls through to the computed `canopycms-settings-<name>`),
|
|
910
1206
|
// so this must never stamp an empty string either way.
|
|
911
|
-
envEntries.push(['CANOPYCMS_SETTINGS_BRANCH', settingsBranch]);
|
|
1207
|
+
envEntries.push(['CANOPYCMS_SETTINGS_BRANCH', settingsBranch, 'worker']);
|
|
912
1208
|
}
|
|
913
|
-
|
|
1209
|
+
if (protectedBranches.length > 0) {
|
|
1210
|
+
// Only the gateway enforces it, and a split worker refuses it.
|
|
1211
|
+
envEntries.push(['CANOPYCMS_PROTECTED_BRANCHES', protectedBranches.join(','), 'gateway']);
|
|
1212
|
+
}
|
|
1213
|
+
const envFile = (readers) => envEntries
|
|
1214
|
+
.filter(([, , reader]) => readers.includes(reader))
|
|
914
1215
|
.map(([name, value]) => `${name}=${assertEnvSafe(name, value)}`)
|
|
915
1216
|
.join('\n');
|
|
916
|
-
const
|
|
1217
|
+
const workerEnv = envFile(['worker', 'both']);
|
|
1218
|
+
const gatewayEnv = envFile(['gateway', 'both']);
|
|
1219
|
+
// After assertEnvSafe, whose leading-quote refusal is the more specific.
|
|
1220
|
+
if (settingsBranch !== undefined)
|
|
1221
|
+
assertGatewayBranchName('settingsBranch', settingsBranch);
|
|
1222
|
+
const efsMountOptions = `tls,iam,accesspoint=${accessPoint.accessPointId}`;
|
|
917
1223
|
const userData = ec2.UserData.forLinux();
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
// marker: a line WITHOUT the timestamp prefix continues the previous
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
1224
|
+
const logGroupName = props.workerLogGroupName ?? `/canopycms/${Stack.of(this).stackName}/worker`;
|
|
1225
|
+
addUserData(userData, [
|
|
1226
|
+
{
|
|
1227
|
+
token: this.workerLogGroup.logGroupName,
|
|
1228
|
+
bytes: Token.isUnresolved(logGroupName)
|
|
1229
|
+
? LOG_GROUP_NAME_MAX_BYTES
|
|
1230
|
+
: Buffer.byteLength(logGroupName),
|
|
1231
|
+
},
|
|
1232
|
+
...envEntries
|
|
1233
|
+
.filter(([, value]) => Token.isUnresolved(value))
|
|
1234
|
+
.map(([, value]) => ({ token: value, bytes: ENV_TOKEN_MAX_BYTES })),
|
|
1235
|
+
], protectedBranches.length, ...withoutShellComments('#!/bin/bash', 'set -euo pipefail', '', '# FAIL-FAST. Every step below is required for the worker to exist at', '# all, except the swap and the package upgrade, which are written as', '# conditions so that they cannot trip it. Until this trap existed, a', '# failure in any required step aborted user-data BEFORE the systemd unit was written -- leaving an instance', "# that runs, passes the ASG's EC2-only health check indefinitely, and", '# does nothing. cfn-signal is deliberately not used here (it would', '# prove nothing about READINESS, which is the argument recorded below),', '# but that argument never covered a boot-SCRIPT failure: `cdk deploy`', '# reported success while publishes queued on EFS, the auth cache went', '# stale and PRs stopped being created, until a human noticed the admin', '# panel showing the worker absent.', '#', '# Shutting down makes the instance fail its EC2 health check, so the ASG', '# replaces it -- which is the only automatic recovery available in this', '# topology. The echo lands in the console log, readable via', '# `aws ec2 get-console-output`, because the CloudWatch agent is itself', '# configured further down and cannot be relied on to exist yet.', 'trap \'echo "canopy-worker user-data FAILED at line $LINENO (exit $?)" >&2; shutdown -h now\' ERR', '', '# Bounded retry for the network-dependent steps. Package mirrors and S3', '# have transient failures; a single flake should cost seconds, not an', '# instance replacement. The first argument names the step in the message', '# an exhausted step leaves on the console.', 'retry() {', ' local step=$1 n=0', ' shift', ' until "$@"; do', ' n=$((n + 1))', ' if [ "$n" -ge 5 ]; then', ` echo "canopy-worker boot: '$step' failed after $n attempts" >&2`, ' return 1', ' fi', ' sleep $((n * 5))', ' done', '}', '', '# Swap, before the first dnf: loading the repository metadata takes more', '# memory than a t4g.nano has free after boot, and the kernel OOM-kills', '# dnf. It stays on afterwards at low swappiness, as headroom for git and', '# the CloudWatch agent rather than working memory. It is on the encrypted', '# root volume, so memory paged out of the worker is encrypted at rest.', '# fallocate is safe here: XFS, the root file system, supports', '# preallocated swap files. A failure only warns, because dnf may still fit', '# and a boot that stopped here would put the group into a replacement', '# loop. Each command carries `|| return 1` because errexit does not apply', '# inside a function called as a condition.', 'setup_swap() {', ' swapon --show=NAME --noheadings | grep -qx /swapfile && return 0', ' if [ "$(stat -c %s /swapfile 2>/dev/null)" != 1073741824 ]; then', ' # The file, plus 2 GiB for packages, logs and the GitHub mirror.', ' local avail_mib', ` avail_mib=$(df --output=avail -m / | tail -n 1 | tr -d ' ')`, ' case "$avail_mib" in ""|*[!0-9]*) return 1 ;; esac', ' if [ "$avail_mib" -lt 3072 ]; then', ' echo "canopy-worker boot: under 3 GiB free on /, so no swap file" >&2', ' return 1', ' fi', ' rm -f /swapfile || return 1', ' fallocate -l 1G /swapfile || return 1', ' fi', ' chmod 600 /swapfile || return 1', ' mkswap /swapfile || return 1', ' # swapon last, so that swap is never on when this returns 1.', " grep -qs '^/swapfile ' /etc/fstab || echo '/swapfile none swap defaults 0 0' >> /etc/fstab || return 1", ' mkdir -p /etc/sysctl.d || return 1', " echo 'vm.swappiness = 10' > /etc/sysctl.d/90-canopy-worker-swap.conf || return 1", ' sysctl -q -w vm.swappiness=10 || return 1', ' swapon /swapfile', '}', 'if ! setup_swap; then', ` echo "canopy-worker boot: 'swap' setup failed; continuing without swap" >&2`, 'fi', '', '# Patch first. AL2023 locks dnf to the repository release its AMI was', '# built from, so a relaunch from the AMI resolved at the last deploy', '# would otherwise install nothing newer than that AMI. Moving to the', '# latest release also makes it the one every install below resolves', '# from. The kernel is left out: a new one applies only at a reboot,', '# which nothing here performs, so it would cost boot time and change', '# nothing. Kernel fixes arrive with the AMI a deploy resolves.', '#', "# A failed upgrade does not stop the boot: the worker starts on the AMI's", '# packages and the next boot (by default the weekly recycle at the', '# latest) tries again. Failing here would put the group into a', '# replacement loop with no worker at all. The installs below then', '# resolve from the AMI release: 2023.7.20250428 carries all of them,', '# 2023.7.20250331 lacks nodejs22. The failure is written to worker.log', '# below, for the unpatched alarm.', 'WORKER_UNPATCHED=0', "if ! retry 'dnf upgrade' dnf upgrade --releasever=latest --exclude='kernel*' -y; then", ' WORKER_UNPATCHED=1', ` echo "canopy-worker boot: ${WORKER_UNPATCHED_LOG_PHRASE}" >&2`, 'fi', '', "retry 'dnf install git' dnf install -y git", "# Node comes from AL2023's own repos, NOT a piped third-party installer.", '# The previous boot curled the NodeSource RPM setup script straight into', '# bash, so every instance replacement -- which the ASG performs on every', '# `cdk deploy`, plus every spot interruption -- depended on a third party', '# being reachable. cms-deploy.test.ts asserts that URL never comes back,', '# which is why it is not spelled out here.', '#', '# nodejs22, not 20: Node 20 reached upstream EOL on 2026-04-30, and this', '# repo declares `engines.node: ">=22"` with .nvmrc `v22`, so the worker', '# was running an EOL runtime BELOW the floor its own code is tested at.', "# 22 (EOL 2027-04-30) matches .nvmrc, CI, and the transform Lambda's", '# NODEJS_22_X, so one runtime is tested everywhere.', '#', '# ExecStart uses the NAMESPACED /usr/bin/node-22 (see workerUnit in', '# worker-units.ts), not the bare `node`: AL2023 installs versioned binaries and', '# points `/usr/bin/node` at one of them through `alternatives`, whose', '# selection AWS documents as able to change at any time.', "retry 'dnf install nodejs22' dnf install -y nodejs22", '', '# Mount EFS through the same access point, at the same path, as the', '# Lambda (see EFS_MOUNT_PATH). amazon-efs-utils refuses an access-point', '# mount without tls (efs_utils_common/mount_options.py).', "retry 'dnf install amazon-efs-utils' dnf install -y amazon-efs-utils", `mkdir -p ${EFS_MOUNT_PATH}`, '# Retried: an `iam` mount also fetches instance-role credentials.', `retry 'EFS mount' mount -t efs -o ${efsMountOptions} ${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH}`, '# Persist the mount across instance reboots: user-data runs once per', '# instance, so without an fstab entry a plain reboot leaves /mnt/efs an', '# empty local dir and the worker would clone a divergent remote.git', '# onto the instance disk, invisible to the Lambda.', `echo '${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH} efs _netdev,${efsMountOptions} 0 0' >> /etc/fstab`, '', '# The bundle runs only if its sha256 is the one given here (synthesized,', '# or the workerCode parameter); a mismatch fails the boot under the trap', '# above.', `retry 'worker bundle download' aws s3 cp s3://${workerBundle.bucketName}/${workerBundle.objectKey} ${WORKER_BUNDLE_DOWNLOAD_PATH}`, `echo '${workerBundle.sha256} ${WORKER_BUNDLE_DOWNLOAD_PATH}' | sha256sum -c -`, '# Root-owned, and read-only to the services (ProtectSystem=strict in worker-units.ts).', `install -D -m 0644 ${WORKER_BUNDLE_DOWNLOAD_PATH} /opt/canopy-worker/index.js`, '# The worker bundle is ESM (esbuild --format=esm). Without this marker a', '# .js file is CommonJS by default and the import statement fails.', '# Node >=22.7 auto-detects module syntax and would mask its absence, and', '# this instance now runs node-22 -- so the marker is kept BECAUSE that', '# auto-detection is a fallback we should not depend on, not because the', '# runtime here still needs it.', `echo '{"type":"module"}' > /opt/canopy-worker/package.json`, '', '# The units and scripts below install from, checked as the bundle is and', '# unpacked where only root can read or write.', `retry 'worker instance files download' aws s3 cp s3://${workerBundle.instanceFiles.bucketName}/${workerBundle.instanceFiles.objectKey} ${INSTANCE_FILES_DOWNLOAD_PATH}`, `echo '${workerBundle.instanceFiles.sha256} ${INSTANCE_FILES_DOWNLOAD_PATH}' | sha256sum -c -`, `install -d -o root -g root -m 0700 ${INSTANCE_FILES_DIR}`, `(cd ${INSTANCE_FILES_DIR} && bash ${INSTANCE_FILES_DOWNLOAD_PATH})`, '', "# The worker's environment: nothing secret.", `cat > ${WORKER_INSTALL_DIR}/.env << '${ENV_HEREDOC_DELIMITER}'`, workerEnv, ENV_HEREDOC_DELIMITER, "# The secrets' names, root's alone until the split gives the gateway's", '# group read below: created with its mode before anything is written to', '# it, so it is never world-readable. The single unit reads both files.', `install -m 0600 -o root -g root /dev/null ${WORKER_INSTALL_DIR}/gateway.env`, `cat > ${WORKER_INSTALL_DIR}/gateway.env << '${ENV_HEREDOC_DELIMITER}'`, gatewayEnv, ENV_HEREDOC_DELIMITER, '', '# The units, chosen by the contract the bundle states in its first', `# lines (\`${WORKER_CONTRACT_BANNER_PREFIX}<n>\`), read after the sha256 check above,`, '# so it is as trusted as the bundle. A bundle at the split contract or', '# later gets the split: a worker with no network and no secret, and the', '# GitHub gateway holding them. An older one gets the single unit it was', '# built for: the split would cut it off from GitHub. Every bundle roll', '# replaces the instance, so each boot matches its own bundle.', 'WORKER_BUNDLE_CONTRACT=$(head -n 5 /opt/canopy-worker/index.js |', ` sed -n 's|^${WORKER_CONTRACT_BANNER_PREFIX}\\([0-9]\\{1,4\\}\\)$|\\1|p' | head -n 1 || true)`, `if [ "\${WORKER_BUNDLE_CONTRACT:-0}" -ge ${GATEWAY_SPLIT_CONTRACT} ]; then`, ' WORKER_SPLIT=1', '', ' # No core dumps on this instance. A piped core_pattern hands a crashing', " # process's memory and environment to a root helper, which journals", " # the environment where the worker's groups can read it; the gateway", ' # refuses to start unless this holds. systemd-sysctl has already run', ' # by now, so the file is applied here as well as at every boot.', ' # fb_tunnels_only_for_init_net keeps fallback tunnel devices out of', " # the worker's network namespace, which must hold only lo. -e skips a", ' # key this kernel lacks: the gateway refuses to start under a piped', ' # core_pattern, and the worker with any interface but lo.', ' mkdir -p /etc/sysctl.d', " cat > /etc/sysctl.d/99-canopy.conf << 'SYSCTLEOF'", 'kernel.core_pattern = core', 'fs.suid_dumpable = 0', 'net.core.fb_tunnels_only_for_init_net = 1', 'SYSCTLEOF', ' sysctl -q -e -p /etc/sysctl.d/99-canopy.conf', ' # PID 1 otherwise gives every service an unlimited core limit', ' # (DefaultLimitCORE=, systemd-system.conf(5)), so', ' # core_pattern=core would leave core files from any crashing daemon.', ' mkdir -p /etc/systemd/system.conf.d', " printf '[Manager]\\nDefaultLimitCORE=0\\n' > /etc/systemd/system.conf.d/canopy.conf", '', " # The gateway's user: a system account, so its uid is stable and", " # gateway.env can be its group's before the service first runs.", ` getent passwd ${GATEWAY_USER} >/dev/null ||`, ` useradd --system --no-create-home --shell /sbin/nologin ${GATEWAY_USER}`, ` chown root:${GATEWAY_USER} ${WORKER_INSTALL_DIR}/gateway.env`, ` chmod 0640 ${WORKER_INSTALL_DIR}/gateway.env`, '', ...['canopy-worker.service', 'canopy-github.service', 'canopy-github.socket'].map((unit) => ` install -m 0644 ${INSTANCE_FILES_DIR}/${unit} /etc/systemd/system/${unit}`), '', ' # The gateway log, root-owned (LOG_RULE, worker-units.ts), and readable', ' # by root alone.', ` install -d -o root -g root -m 0755 ${GATEWAY_LOG_DIR}`, ` install -o root -g root -m 0640 /dev/null ${GATEWAY_LOG}`, '', ' # canopy-inspect, for an operator on EFS: see canopyInspectScript', ' # (worker-units.ts).', ` install -m 0755 ${INSTANCE_FILES_DIR}/canopy-inspect ${CANOPY_INSPECT}`, ` install -m 0755 ${INSTANCE_FILES_DIR}/inspect-checks ${INSPECT_CHECKS}`, ` install -m 0644 ${INSTANCE_FILES_DIR}/inspect.rc ${INSPECT_RC}`, 'else', ' WORKER_SPLIT=0', ` install -m 0644 ${INSTANCE_FILES_DIR}/canopy-worker-single-process.service /etc/systemd/system/canopy-worker.service`, 'fi', '', '# The worker log, root-owned and created before the first start (LOG_RULE,', '# worker-units.ts). 0644, so it stays', '# readable without sudo.', `install -d -o root -g root -m 0755 ${WORKER_LOG_DIR}`, `install -o root -g root -m 0644 /dev/null ${WORKER_LOG}`, '', '# In the worker log format (canopycms worker/log.ts), so the CloudWatch', '# agent ships it as its own event: the agent reads a file it has no saved', '# position for from the start (its `from_beginning` defaults to true).', 'if [ "$WORKER_UNPATCHED" = 1 ]; then', ` echo "$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) ERROR canopy-worker boot: 'dnf upgrade' failed; ${WORKER_UNPATCHED_LOG_PHRASE}" >> ${WORKER_LOG}`, 'fi', '', '# Start the units; daemon-reload also reads the core-limit drop-in. The', '# gateway first, so its secret reads and first GitHub fetch start before', "# the worker's first call.", 'systemctl daemon-reload', 'if [ "$WORKER_SPLIT" = 1 ]; then', ' systemctl enable --now canopy-github.socket canopy-github.service', 'fi', 'systemctl enable canopy-worker', 'systemctl start canopy-worker', '', '# ---- CloudWatch log shipping ----', '# Placed AFTER worker start: with set -euo pipefail, a package/agent', '# failure here must not prevent the worker from running (shipping is', '# best-effort).', '#', '# DISARM THE FAIL-FAST TRAP AND errexit FIRST. Everything above this line', '# is required for the worker to exist at all, so a failure there should', '# replace the instance. Nothing below it is: the worker is already', '# running and healthy by this point. Leaving the trap armed would let a', '# package-mirror outage during the agent install shut down a perfectly', '# good worker -- and the ASG would relaunch straight into the same', '# outage, turning degraded log shipping into a replacement loop with the', '# worker down for its duration. That is strictly worse than the', '# best-effort behaviour this section has always documented.', 'trap - ERR', 'set +e', "retry 'dnf install amazon-cloudwatch-agent logrotate' dnf install -y amazon-cloudwatch-agent logrotate", '', '# Bound on-disk growth; copytruncate keeps the fd the CW agent tails valid', '# (tiny copy->truncate loss window is acceptable for diagnostic logs).', `cat > /etc/logrotate.d/canopy-worker << 'ROTEOF'`, `${WORKER_LOG} ${GATEWAY_LOG} {`, ' size 10M', ' rotate 5', ' compress', ' copytruncate', ' missingok', ' notifempty', '}', 'ROTEOF', '# The config above only fires when logrotate actually runs; AL2023', '# presets may leave logrotate.timer disabled, and without it the size', '# cap never triggers and worker.log grows until the nano disk fills.', '# --now is idempotent if the timer is already enabled/running.', 'systemctl enable --now logrotate.timer', '', '# No retention_in_days here: CDK owns retention on the pre-created group.', `cat > /opt/aws/amazon-cloudwatch-agent/etc/canopy-worker-logs.json << 'CWEOF'`, '{', ' "logs": {', ' "logs_collected": {', ' "files": {', ' "collect_list": [', ...[
|
|
1236
|
+
{ file: WORKER_LOG, stream: '{instance_id}' },
|
|
1237
|
+
// The gateway's own record of what reached GitHub, which the worker
|
|
1238
|
+
// cannot write.
|
|
1239
|
+
{ file: GATEWAY_LOG, stream: '{instance_id}-gateway' },
|
|
1240
|
+
].flatMap(({ file, stream }, i, all) => [
|
|
1241
|
+
' {',
|
|
1242
|
+
` "file_path": "${file}",`,
|
|
1243
|
+
` "log_group_name": "${this.workerLogGroup.logGroupName}",`,
|
|
1244
|
+
` "log_stream_name": "${stream}",`,
|
|
1245
|
+
// Both prefix every line with an ISO-8601 timestamp
|
|
1246
|
+
// (packages/canopycms/src/worker/log.ts). Parsing it here is what makes
|
|
1247
|
+
// CloudWatch show the time the process emitted a line rather than the
|
|
1248
|
+
// time the agent shipped it - those diverge exactly when it matters
|
|
1249
|
+
// (agent hiccup, buffered burst, post-restart backlog).
|
|
1250
|
+
' "timestamp_format": "%Y-%m-%dT%H:%M:%S.%f",',
|
|
1251
|
+
// The prefix ends in `Z`, so the parsed time is UTC. Without this the
|
|
1252
|
+
// agent would interpret it in the instance's local zone.
|
|
1253
|
+
' "timezone": "UTC",',
|
|
1254
|
+
// "{timestamp_format}" reuses the pattern above as the multi-line start
|
|
1255
|
+
// marker: a line WITHOUT the timestamp prefix continues the previous
|
|
1256
|
+
// event instead of becoming its own. That is what keeps a stack trace as
|
|
1257
|
+
// one CloudWatch event, and why every writer to these files must go
|
|
1258
|
+
// through the worker log helpers.
|
|
1259
|
+
' "multi_line_start_pattern": "{timestamp_format}"',
|
|
1260
|
+
i < all.length - 1 ? ' },' : ' }',
|
|
1261
|
+
]), ' ]', ' }', ' }', ' }', '}', 'CWEOF', '# fetch-config -s starts AND systemctl-enables the agent (reboot-persistent).', '/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/etc/canopy-worker-logs.json'));
|
|
934
1262
|
// Launch template (not the deprecated AutoScalingGroup instanceType/
|
|
935
1263
|
// machineImage/... shorthand): that shorthand synthesizes an
|
|
936
1264
|
// AWS::AutoScaling::LaunchConfiguration, which AWS accounts created after
|
|
@@ -938,24 +1266,60 @@ export class CanopyCmsService extends Construct {
|
|
|
938
1266
|
// fresh adopter account. An explicit LaunchTemplate synthesizes
|
|
939
1267
|
// AWS::EC2::LaunchTemplate instead, which every account can use.
|
|
940
1268
|
const launchTemplate = new ec2.LaunchTemplate(this, 'WorkerLaunchTemplate', {
|
|
941
|
-
instanceType:
|
|
1269
|
+
instanceType: workerInstanceTypes[0],
|
|
942
1270
|
machineImage: ec2.MachineImage.latestAmazonLinux2023({
|
|
943
1271
|
cpuType: ec2.AmazonLinuxCpuType.ARM_64,
|
|
944
1272
|
}),
|
|
945
1273
|
role: workerRole,
|
|
946
1274
|
securityGroup: workerSg,
|
|
947
1275
|
userData,
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
1276
|
+
requireImdsv2: true,
|
|
1277
|
+
// 1, not AL2023's 2: everything that reads metadata runs on the host,
|
|
1278
|
+
// and a second hop only serves containers.
|
|
1279
|
+
httpPutResponseHopLimit: 1,
|
|
1280
|
+
blockDevices: [
|
|
1281
|
+
{
|
|
1282
|
+
// AL2023's root device. Its size is removed below.
|
|
1283
|
+
deviceName: '/dev/xvda',
|
|
1284
|
+
volume: ec2.BlockDeviceVolume.ebs(8, {
|
|
1285
|
+
encrypted: true,
|
|
1286
|
+
volumeType: ec2.EbsDeviceVolumeType.GP3,
|
|
1287
|
+
deleteOnTermination: true,
|
|
1288
|
+
}),
|
|
1289
|
+
},
|
|
1290
|
+
],
|
|
952
1291
|
});
|
|
1292
|
+
launchTemplate.node.defaultChild.addPropertyDeletionOverride('LaunchTemplateData.BlockDeviceMappings.0.Ebs.VolumeSize');
|
|
953
1293
|
this.workerAsg = new autoscaling.AutoScalingGroup(this, 'WorkerAsg', {
|
|
954
1294
|
vpc: this.vpc,
|
|
1295
|
+
// Public, deliberately: it reaches GitHub and AWS APIs through the
|
|
1296
|
+
// internet gateway instead of a NAT gateway that would cost more than
|
|
1297
|
+
// the rest of the stack. Its security group admits no inbound traffic.
|
|
955
1298
|
vpcSubnets: { subnetType: ec2.SubnetType.PUBLIC },
|
|
956
|
-
|
|
1299
|
+
// Spot goes through a mixed-instances policy rather than spot options on
|
|
1300
|
+
// the launch template, so the group can choose among several pools.
|
|
1301
|
+
...(workerCapacity.type === 'spot'
|
|
1302
|
+
? {
|
|
1303
|
+
mixedInstancesPolicy: {
|
|
1304
|
+
launchTemplate,
|
|
1305
|
+
launchTemplateOverrides: workerInstanceTypes.map((instanceType) => ({
|
|
1306
|
+
instanceType,
|
|
1307
|
+
})),
|
|
1308
|
+
instancesDistribution: {
|
|
1309
|
+
onDemandBaseCapacity: 0,
|
|
1310
|
+
onDemandPercentageAboveBaseCapacity: 0,
|
|
1311
|
+
spotAllocationStrategy: autoscaling.SpotAllocationStrategy.PRICE_CAPACITY_OPTIMIZED,
|
|
1312
|
+
spotMaxPrice: workerCapacity.maxPrice,
|
|
1313
|
+
},
|
|
1314
|
+
},
|
|
1315
|
+
capacityRebalance: workerCapacity.capacityRebalance ?? true,
|
|
1316
|
+
}
|
|
1317
|
+
: { launchTemplate }),
|
|
957
1318
|
minCapacity: 1,
|
|
958
1319
|
maxCapacity: 1,
|
|
1320
|
+
maxInstanceLifetime: props.workerMaxInstanceLifetime === null
|
|
1321
|
+
? undefined
|
|
1322
|
+
: (props.workerMaxInstanceLifetime ?? Duration.days(7)),
|
|
959
1323
|
// `healthChecks`, not the deprecated `healthCheck`/`HealthCheck.ec2({ grace })`:
|
|
960
1324
|
// both synthesize the same HealthCheckType/HealthCheckGracePeriod, but the
|
|
961
1325
|
// deprecated form prints a jsii warning on every synth -- in adopters'
|
|
@@ -966,10 +1330,10 @@ export class CanopyCmsService extends Construct {
|
|
|
966
1330
|
// Without an updatePolicy, CloudFormation's default behavior for an ASG
|
|
967
1331
|
// behind a changed launch template is to update the template resource
|
|
968
1332
|
// and do NOTHING else - the running instance keeps its old user-data
|
|
969
|
-
// (and therefore the old worker code: the
|
|
970
|
-
//
|
|
971
|
-
//
|
|
972
|
-
//
|
|
1333
|
+
// (and therefore the old worker code: the bundle's key and sha256,
|
|
1334
|
+
// from the CDK asset or the workerCode parameter, are interpolated into
|
|
1335
|
+
// user data) until an interruption or a manual terminate happens to
|
|
1336
|
+
// replace it. `cdk deploy` would then silently deploy everything
|
|
973
1337
|
// EXCEPT the worker. `rollingUpdate` makes CloudFormation actually
|
|
974
1338
|
// terminate-and-relaunch the instance on every deploy that changes the
|
|
975
1339
|
// launch template, so a worker code change actually reaches it.
|
|
@@ -982,10 +1346,14 @@ export class CanopyCmsService extends Construct {
|
|
|
982
1346
|
// That outage is acceptable: the task queue and branch workspaces live on
|
|
983
1347
|
// EFS, not on the instance, so the new instance picks up where the old one
|
|
984
1348
|
// left off, and the Lambda's Save/Publish paths only enqueue task files
|
|
985
|
-
// onto EFS and never talk to the worker directly.
|
|
986
|
-
//
|
|
987
|
-
//
|
|
988
|
-
//
|
|
1349
|
+
// onto EFS and never talk to the worker directly. The old worker drains
|
|
1350
|
+
// first, behind the terminating lifecycle hook below. Orphan recovery on
|
|
1351
|
+
// every task-queue cycle (CmsWorker.processTaskQueue) covers a worker
|
|
1352
|
+
// that died without draining.
|
|
1353
|
+
//
|
|
1354
|
+
// Single-writer does not rest on this ordering: the worker lock on EFS
|
|
1355
|
+
// keeps a second worker from starting until the first has released it,
|
|
1356
|
+
// which a draining worker does last.
|
|
989
1357
|
//
|
|
990
1358
|
// `waitOnResourceSignals` therefore defaults to false here - see the
|
|
991
1359
|
// no-cfn-signal note below.
|
|
@@ -1014,6 +1382,42 @@ export class CanopyCmsService extends Construct {
|
|
|
1014
1382
|
// `mount -t efs` failure kills the whole bootstrap and the
|
|
1015
1383
|
// EC2-health-checked ASG never notices.
|
|
1016
1384
|
this.workerAsg.node.addDependency(this.fileSystem.mountTargetsAvailable);
|
|
1385
|
+
// Holds a terminating instance in Terminating:Wait until its worker has
|
|
1386
|
+
// drained and completed the action (worker/termination-watch.ts). CONTINUE
|
|
1387
|
+
// on timeout, so a hung worker delays termination by at most the heartbeat.
|
|
1388
|
+
//
|
|
1389
|
+
// A separate resource that depends on the group, so an upgrade that adds it
|
|
1390
|
+
// rolls the group first: the outgoing instance, whose worker predates the
|
|
1391
|
+
// drain, is never held for a heartbeat it cannot complete.
|
|
1392
|
+
this.workerAsg.addLifecycleHook('WorkerDrainHook', {
|
|
1393
|
+
lifecycleHookName: WORKER_DRAIN_HOOK_NAME,
|
|
1394
|
+
lifecycleTransition: autoscaling.LifecycleTransition.INSTANCE_TERMINATING,
|
|
1395
|
+
heartbeatTimeout: workerTerminationHeartbeat,
|
|
1396
|
+
defaultResult: autoscaling.DefaultResult.CONTINUE,
|
|
1397
|
+
});
|
|
1398
|
+
// A standalone policy rather than the role's default one: it names the
|
|
1399
|
+
// group, and the group depends (through the launch template) on the role.
|
|
1400
|
+
new iam.Policy(this, 'WorkerLifecyclePolicy', {
|
|
1401
|
+
roles: [workerRole],
|
|
1402
|
+
statements: [
|
|
1403
|
+
new iam.PolicyStatement({
|
|
1404
|
+
// Describe actions support no resource-level permissions.
|
|
1405
|
+
actions: ['autoscaling:DescribeAutoScalingInstances'],
|
|
1406
|
+
resources: ['*'],
|
|
1407
|
+
}),
|
|
1408
|
+
new iam.PolicyStatement({
|
|
1409
|
+
actions: ['autoscaling:CompleteLifecycleAction'],
|
|
1410
|
+
resources: [
|
|
1411
|
+
Stack.of(this).formatArn({
|
|
1412
|
+
service: 'autoscaling',
|
|
1413
|
+
resource: 'autoScalingGroup',
|
|
1414
|
+
resourceName: `*:autoScalingGroupName/${this.workerAsg.autoScalingGroupName}`,
|
|
1415
|
+
arnFormat: ArnFormat.COLON_RESOURCE_NAME,
|
|
1416
|
+
}),
|
|
1417
|
+
],
|
|
1418
|
+
}),
|
|
1419
|
+
],
|
|
1420
|
+
});
|
|
1017
1421
|
}
|
|
1018
1422
|
/**
|
|
1019
1423
|
* Serve the editor from a CloudFront distribution you already own, such as
|