canopycms-cdk 0.0.68-int.98 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,16 +1,10 @@
1
- import * as path from 'node:path';
2
- import { fileURLToPath } from 'node:url';
3
1
  import { Construct } from 'constructs';
4
- import { Annotations, Duration, RemovalPolicy, Stack, Token, aws_ec2 as ec2, aws_efs as efs, aws_iam as iam, aws_lambda as lambda, aws_autoscaling as autoscaling, aws_s3_assets as s3assets, aws_logs as logs, } from 'aws-cdk-lib';
2
+ import { Annotations, ArnFormat, Duration, Names, RemovalPolicy, Stack, Token, Tokenization, aws_cloudwatch as cloudwatch, aws_cloudwatch_actions as cloudwatchActions, aws_ec2 as ec2, aws_efs as efs, aws_iam as iam, aws_lambda as lambda, aws_autoscaling as autoscaling, aws_logs as logs, } from 'aws-cdk-lib';
5
3
  import { attachLambdaExecutionPolicies } from './lambda-execution-role.js';
4
+ import { INSTANCE_FILES_DOWNLOAD_PATH, WORKER_BUNDLE_DOWNLOAD_PATH, workerBundleSource, } from './worker-bundle.js';
6
5
  import { attachEditorBehaviors } from './editor-routing.js';
7
- // This package (`canopycms-cdk`) is `"type": "module"`, so its compiled output
8
- // is real ESM and `__dirname` is not a global there - the worker asset path
9
- // below throws `__dirname is not defined` under a real ESM runtime (e.g. `tsx`)
10
- // without this. Vitest's SSR/CJS-interop transform shims `__dirname`
11
- // automatically, so this file's own tests would not catch its absence. Same fix
12
- // as ../../lambda/asset-transform/build.mjs and ./asset-support.ts.
13
- const __dirname = path.dirname(fileURLToPath(import.meta.url));
6
+ import { GATEWAY_SPLIT_CONTRACT, WORKER_CAPACITY_ENV, WORKER_CONTRACT_BANNER_PREFIX, WORKER_DRAIN_HOOK_NAME, WORKER_SYNC_LOG_PHRASE, } from './worker-lifecycle.js';
7
+ import { CANOPY_INSPECT, EFS_MOUNT_PATH, GATEWAY_LOG, GATEWAY_LOG_DIR, GATEWAY_USER, INSPECT_CHECKS, INSPECT_RC, INSTANCE_FILES_DIR, WORKER_INSTALL_DIR, WORKER_LOG, WORKER_LOG_DIR, } from './worker-units.js';
14
8
  /**
15
9
  * Synth-time mirror of `resolveDeploymentName`'s rule in the `canopycms`
16
10
  * package (packages/canopycms/src/operating-mode/deployment-name.ts).
@@ -18,7 +12,7 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
18
12
  * Duplicated rather than imported so a CONSTRUCT-only consumer does not pay for
19
13
  * the core package's module graph. Whether that still justifies duplicating is
20
14
  * an open question, tracked alongside the same question about the S3 prefix
21
- * constants in .claude/future-tasks/cdk-prefixes-duplication.md.
15
+ * constants in safeinsights/canopycms#608.
22
16
  *
23
17
  * Drift between the two copies is caught by a test, not by this comment:
24
18
  * cms-deploy.test.ts drives both this construct and the runtime predicate over
@@ -36,21 +30,18 @@ const isValidDeploymentName = (name) => /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(name
36
30
  !name.endsWith('.') &&
37
31
  !name.endsWith('.lock');
38
32
  /**
39
- * Where the Lambda AND the worker mount the `WorkspaceAP` access point (rooted
40
- * at EFS:/workspace), and so the workspace root of both.
41
- *
42
- * One path string, not merely one directory: git records absolute paths on
43
- * EFS (a workspace clone's `origin` is `<root>/remote.git`), and a path written
44
- * by one process must resolve for the other.
33
+ * The text of the worker.log line user data writes when the boot's package
34
+ * upgrade failed and the worker runs on the AMI's packages. The unpatched
35
+ * alarm's metric filter counts it.
45
36
  */
46
- const EFS_MOUNT_PATH = '/mnt/efs';
37
+ const WORKER_UNPATCHED_LOG_PHRASE = 'running unpatched on the AMI packages until the next boot';
47
38
  /**
48
- * The heredoc delimiter user-data uses to write the worker's `.env` (see the
49
- * `cat > … << 'ENVEOF'` block below).
39
+ * The heredoc delimiter user-data uses to write the worker's `.env` and the
40
+ * gateway's `gateway.env` (see the `cat > … << 'ENVEOF'` blocks below).
50
41
  */
51
42
  const ENV_HEREDOC_DELIMITER = 'ENVEOF';
52
43
  /**
53
- * Guard every value interpolated into the worker's `.env` file.
44
+ * Guard every value interpolated into an env file (`.env` or `gateway.env`).
54
45
  *
55
46
  * This is robustness, not a security boundary: the heredoc delimiter is
56
47
  * quoted, so the shell performs no expansion on the body, and these values
@@ -68,7 +59,7 @@ const ENV_HEREDOC_DELIMITER = 'ENVEOF';
68
59
  function assertEnvSafe(name, value) {
69
60
  if (/[\r\n]/.test(value)) {
70
61
  throw new Error(`CanopyCmsService: ${name} must not contain a newline (got ${JSON.stringify(value)}). ` +
71
- `It is written into the worker's .env file, where a newline injects an arbitrary ` +
62
+ `It is written into an env file (.env or gateway.env), where a newline injects an arbitrary ` +
72
63
  `extra environment line.`);
73
64
  }
74
65
  // systemd (EnvironmentFile=, see the unit below) treats a value whose FIRST
@@ -79,14 +70,14 @@ function assertEnvSafe(name, value) {
79
70
  // which is why this checks position 0 rather than banning the character.
80
71
  if (value.startsWith('"') || value.startsWith("'")) {
81
72
  throw new Error(`CanopyCmsService: ${name} must not start with a quote character ` +
82
- `(got ${JSON.stringify(value)}). It is written into the worker's .env file, which ` +
73
+ `(got ${JSON.stringify(value)}). It is written into an env file (.env or gateway.env), which ` +
83
74
  `systemd reads as EnvironmentFile -- a leading quote opens a quoted value that ` +
84
75
  `consumes every following line until a matching quote, silently emptying the rest ` +
85
- `of the worker's environment.`);
76
+ `of the environment.`);
86
77
  }
87
78
  if (value.includes(ENV_HEREDOC_DELIMITER)) {
88
79
  throw new Error(`CanopyCmsService: ${name} must not contain ${JSON.stringify(ENV_HEREDOC_DELIMITER)} ` +
89
- `(got ${JSON.stringify(value)}). It is written into the worker's .env file with a ` +
80
+ `(got ${JSON.stringify(value)}). It is written into an env file (.env or gateway.env) with a ` +
90
81
  `<< '${ENV_HEREDOC_DELIMITER}' heredoc, which that value would terminate early.`);
91
82
  }
92
83
  // The same systemd `EnvironmentFile=` parser the leading-quote rule above is
@@ -97,7 +88,7 @@ function assertEnvSafe(name, value) {
97
88
  // in -- so it is refused here rather than debugged on an instance.
98
89
  if (value.includes('\\')) {
99
90
  throw new Error(`CanopyCmsService: ${name} must not contain a backslash (got ${JSON.stringify(value)}). ` +
100
- `It is written into the worker's .env file, which systemd reads as EnvironmentFile -- ` +
91
+ `It is written into an env file (.env or gateway.env), which systemd reads as EnvironmentFile -- ` +
101
92
  `there a backslash escapes the next character, and a trailing one continues the value ` +
102
93
  `onto the following line, consuming the next variable entirely.`);
103
94
  }
@@ -107,11 +98,74 @@ function assertEnvSafe(name, value) {
107
98
  // by a route no charset check upstream can see.
108
99
  if (value !== value.trim()) {
109
100
  throw new Error(`CanopyCmsService: ${name} must not start or end with whitespace ` +
110
- `(got ${JSON.stringify(value)}). systemd strips it when reading the worker's .env, so ` +
111
- `the value the worker sees would differ from the one configured here.`);
101
+ `(got ${JSON.stringify(value)}). systemd strips it when reading the env file, so ` +
102
+ `the value the service sees would differ from the one configured here.`);
112
103
  }
113
104
  return value;
114
105
  }
106
+ /**
107
+ * `lines` without their shell comment lines, outside heredoc bodies, so the comments stay in this
108
+ * file and out of the 16 KiB EC2 allows user data. A `#!` line is kept, and every line of a
109
+ * heredoc is kept as written. Only the quoted form, a line ending `<< 'DELIM'`, may open one: any
110
+ * other `<<` is refused, since its body would be read as commands and comments here.
111
+ * @internal Exported for tests.
112
+ */
113
+ export function withoutShellComments(...lines) {
114
+ let heredocEnd;
115
+ return lines.filter((line) => {
116
+ if (heredocEnd !== undefined) {
117
+ if (line === heredocEnd)
118
+ heredocEnd = undefined;
119
+ return true;
120
+ }
121
+ if (/^\s*#(?!!)/.test(line))
122
+ return false;
123
+ if (line.includes('<<')) {
124
+ heredocEnd = /<< '(\w+)'$/.exec(line)?.[1];
125
+ if (heredocEnd === undefined)
126
+ throw new Error(`user data: unquoted heredoc in ${line}`);
127
+ }
128
+ return true;
129
+ });
130
+ }
131
+ /** EC2's limit on user data, in bytes, before base64. */
132
+ const USER_DATA_MAX_BYTES = 16_384;
133
+ /**
134
+ * The most bytes a deploy-time value the construct itself puts in user data resolves to: a bucket
135
+ * name is at most 63, an asset key or a sha256 under 80, a file system or access point id under 25.
136
+ */
137
+ const USER_DATA_TOKEN_MAX_BYTES = 128;
138
+ /**
139
+ * The most bytes a deploy-time env value from a prop resolves to: a Secrets Manager ARN with a
140
+ * 512-character name is about 600. A longer token (an SSM parameter's value) is the adopter's to
141
+ * keep short.
142
+ */
143
+ const ENV_TOKEN_MAX_BYTES = 640;
144
+ /** The most bytes a CloudWatch log group's name can be. */
145
+ const LOG_GROUP_NAME_MAX_BYTES = 512;
146
+ /**
147
+ * Add `lines` to `userData`, refusing at synth a configuration whose user data could exceed what
148
+ * EC2 accepts (EC2 would refuse the launch template, or every launch, and the group would never
149
+ * get a worker). Each token counts as its `bounds` entry's bytes, or
150
+ * {@link USER_DATA_TOKEN_MAX_BYTES}; an env value from a token longer than
151
+ * {@link ENV_TOKEN_MAX_BYTES} is not caught.
152
+ */
153
+ function addUserData(userData, bounds, protectedBranchCount, ...lines) {
154
+ userData.addCommands(...lines);
155
+ const bytesOf = new Map(bounds.flatMap(({ token, bytes }) => Tokenization.reverseString(token).tokens.map((resolvable) => [resolvable, bytes])));
156
+ const longest = Tokenization.reverseString(userData.render())
157
+ .mapTokens({
158
+ mapToken: (token) => 'x'.repeat(bytesOf.get(token) ?? USER_DATA_TOKEN_MAX_BYTES),
159
+ })
160
+ .join({ join: (left, right) => `${String(left)}${String(right)}` });
161
+ const bytes = Buffer.byteLength(String(longest));
162
+ if (bytes > USER_DATA_MAX_BYTES) {
163
+ throw new Error(`CanopyCmsService: the worker's user data could reach ${bytes} bytes, over the ` +
164
+ `${USER_DATA_MAX_BYTES} EC2 accepts; protectedBranches has ${protectedBranchCount} ` +
165
+ `entries. Protect branches with fewer, wider * patterns, or shorten the other names ` +
166
+ `passed in (secret ARNs and fields, branches, the repository, the worker log group).`);
167
+ }
168
+ }
115
169
  /**
116
170
  * Refuse a `baseBranch`/`settingsBranch` that git itself would reject, at
117
171
  * synth rather than at worker boot.
@@ -130,9 +184,10 @@ function assertEnvSafe(name, value) {
130
184
  * would refuse `cdk synth` for an adopter whose default branch is `release/v2`,
131
185
  * and the worker handles such names fine - it keeps the raw name for git refs
132
186
  * and runs it through `sanitizeBranchName` only for workspace DIRECTORY names.
133
- * This implements git's `check-ref-format` rules for a branch name instead.
134
- * Nothing in `canopycms` validates a branch name, so there is no runtime
135
- * counterpart to drift from and no shared fixture list like `deploymentName`'s.
187
+ * This implements git's `check-ref-format` rules for a branch name instead,
188
+ * plus the GitHub gateway's 255-character cap: the gateway refuses a longer
189
+ * pull request base. A settings branch is pushed through the gateway, so it
190
+ * takes the gateway's narrower name rules too (`assertGatewayBranchName`).
136
191
  */
137
192
  function assertValidGitBranchName(propName, value) {
138
193
  const reject = (why) => {
@@ -145,6 +200,8 @@ function assertValidGitBranchName(propName, value) {
145
200
  };
146
201
  if (value.length === 0)
147
202
  reject('it is empty');
203
+ if (value.length > 255)
204
+ reject('it is longer than 255 characters');
148
205
  if (value === '@')
149
206
  reject("a lone '@' is reserved by git");
150
207
  // `git check-ref-format --branch` rejects HEAD: it names the symbolic ref,
@@ -182,6 +239,67 @@ function assertValidGitBranchName(propName, value) {
182
239
  }
183
240
  return value;
184
241
  }
242
+ /**
243
+ * Refuse a name the GitHub gateway would refuse to push (`branchNameProblem`,
244
+ * packages/canopycms/src/worker/branch-policy.ts), on top of git's rules: printable ASCII, no
245
+ * `"`, `%`, `{` or `}`, no leading `refs/`, `heads/`, `tags/` or `remotes/`.
246
+ */
247
+ function assertGatewayBranchName(propName, value) {
248
+ const reject = (why) => {
249
+ throw new Error(`CanopyCmsService: invalid ${propName} ${JSON.stringify(value)} -- ${why}.`);
250
+ };
251
+ if (!/^[\x20-\x7e]*$/.test(value))
252
+ reject('it is not printable ASCII');
253
+ if (/["%{}]/.test(value))
254
+ reject('it contains ", %, { or }');
255
+ if (/^(?:refs|heads|tags|remotes)\//i.test(value)) {
256
+ reject("it starts with 'refs/', 'heads/', 'tags/' or 'remotes/'");
257
+ }
258
+ return value;
259
+ }
260
+ /**
261
+ * The most `protectedBranches` entries the gateway takes
262
+ * (`MAX_PROTECTED_BRANCHES`, packages/canopycms/src/worker/branch-policy.ts). A local constant:
263
+ * this module imports nothing from core.
264
+ */
265
+ const MAX_PROTECTED_BRANCHES = 1024;
266
+ /**
267
+ * Refuse `protectedBranches` the GitHub gateway would refuse at boot, which would crash-loop it
268
+ * (`protectedPatternProblem`, packages/canopycms/src/worker/branch-policy.ts): more than
269
+ * {@link MAX_PROTECTED_BRANCHES} entries, and any entry that is not a branch name in which `*` is
270
+ * a wildcard, held to git's rules and the gateway's (`assertGatewayBranchName`). No entry may
271
+ * start with the reserved settings prefix: a glob there matches nothing, and a name there would
272
+ * stop the deployment's settings reaching GitHub, since a settings branch is pushed plainly every
273
+ * sync. A comma separates entries in the env file, so none may hold one.
274
+ */
275
+ function assertValidProtectedBranches(entries) {
276
+ if (entries.length > MAX_PROTECTED_BRANCHES) {
277
+ throw new Error(`CanopyCmsService: protectedBranches has ${entries.length} entries; the most it takes is ${MAX_PROTECTED_BRANCHES}.`);
278
+ }
279
+ return entries.map((entry) => {
280
+ const reject = (why) => {
281
+ throw new Error(`CanopyCmsService: invalid protectedBranches entry ${JSON.stringify(entry)} -- ${why}.`);
282
+ };
283
+ if (entry.includes(','))
284
+ reject('it contains a comma, which separates entries');
285
+ assertGatewayBranchName('protectedBranches entry', entry);
286
+ if (entry.startsWith('canopycms-settings-')) {
287
+ reject(entry.includes('*')
288
+ ? 'the worker never lets a glob match a settings branch, so this one would match nothing'
289
+ : "a settings branch is pushed plainly every sync, so protecting it would stop this deployment's settings reaching GitHub");
290
+ }
291
+ const asName = entry.replace(/\*/g, 'x');
292
+ try {
293
+ assertValidGitBranchName('protectedBranches entry', asName);
294
+ }
295
+ catch (err) {
296
+ // Name the entry as written, not with its wildcards replaced.
297
+ const message = err instanceof Error ? err.message : String(err);
298
+ throw new Error(message.replace(JSON.stringify(asName), JSON.stringify(entry)));
299
+ }
300
+ return entry;
301
+ });
302
+ }
185
303
  /**
186
304
  * A Secrets Manager ARN carrying the ECS/CloudFormation JSON-field suffix,
187
305
  * i.e. `arn:…:secret:name-AbCdEf:MY_KEY::` rather than `arn:…:secret:name-AbCdEf`.
@@ -317,7 +435,7 @@ const GITHUB_TOKEN_PROP_NAMES = ['githubTokenSecretArn', 'githubTokenSecretJsonF
317
435
  * Rejects a PEM private key passed where a prop expects an identifier or an ARN.
318
436
  *
319
437
  * There is no plaintext private-key prop, and there cannot be one: the value
320
- * would be written into the worker's `.env`, which systemd reads as
438
+ * would be written into `gateway.env`, which systemd reads as
321
439
  * `EnvironmentFile=` where a newline begins a new variable. So the realistic
322
440
  * mistake is to paste the key into `githubAppPrivateKeySecretArn` -- the prop
323
441
  * whose name contains "PrivateKey" -- instead of the ARN of a secret holding it.
@@ -346,7 +464,7 @@ function assertNotInlinePrivateKey(propName, value) {
346
464
  return;
347
465
  throw new Error(`CanopyCmsService: ${propName} looks like a PEM private key, not ${propName === 'githubAppPrivateKeySecretArn' ? 'a secret ARN' : 'an identifier'}. ` +
348
466
  `The GitHub App private key can ONLY be supplied as a Secrets Manager ARN -- it is ` +
349
- `multi-line, and every value this construct configures goes into the worker's .env file, ` +
467
+ `multi-line, and every value this construct configures goes into an env file (.env or gateway.env), ` +
350
468
  `which systemd reads as EnvironmentFile where a newline starts a new variable. Store the ` +
351
469
  `PEM in Secrets Manager and pass that secret's full ARN as githubAppPrivateKeySecretArn ` +
352
470
  `(optionally with githubAppPrivateKeySecretJsonField if it lives inside a JSON document).`);
@@ -430,7 +548,7 @@ function assertGitHubAuthProps(props) {
430
548
  `drop them and use githubTokenSecretArn.`);
431
549
  }
432
550
  // The JSON field counts as "a token is configured", not just the ARN. An
433
- // adopter following docs/adopter-migration.md removes `githubTokenSecretArn`
551
+ // adopter following CHANGELOG.md removes `githubTokenSecretArn`
434
552
  // and overlooks `githubTokenSecretJsonField`; left out of this check, that
435
553
  // lands on `assertSecretPropPair` instead, which answers "Set
436
554
  // githubTokenSecretArn, or drop githubTokenSecretJsonField" -- pointing them
@@ -474,6 +592,31 @@ export const DEFAULT_CMS_LAMBDA_TIMEOUT = Duration.seconds(60);
474
592
  export const MIN_CMS_RESERVED_CONCURRENCY = 20;
475
593
  /** Default `reservedConcurrency`; see that prop for what it caps and costs. */
476
594
  export const DEFAULT_CMS_RESERVED_CONCURRENCY = 50;
595
+ const DEFAULT_SPOT_INSTANCE_TYPES = ['t4g.nano', 't4g.micro', 't4g.small'];
596
+ /** The worker's internal drain deadline (core's DEFAULT_DRAIN_DEADLINE_MS) plus two minutes. */
597
+ export const MIN_WORKER_TERMINATION_HEARTBEAT = Duration.seconds(210);
598
+ const MAX_WORKER_TERMINATION_HEARTBEAT = Duration.hours(2);
599
+ /** Graviton families: `a1`, or a generation digit followed by `g` (t4g, c7gn, m8gd). */
600
+ function assertGravitonInstanceType(instanceType) {
601
+ const name = instanceType.toString();
602
+ if (Token.isUnresolved(name))
603
+ return;
604
+ if (!/^(a1|[a-z]+\d+g[a-z]*)\./.test(name)) {
605
+ throw new Error(`workerCapacity: instance type "${name}" is not Graviton (arm64). The worker's AMI is ` +
606
+ 'Amazon Linux 2023 for arm64, so an x86 instance cannot boot it.');
607
+ }
608
+ }
609
+ function assertWorkerTerminationHeartbeat(heartbeat) {
610
+ if (Token.isUnresolved(heartbeat.toSeconds({ integral: false })))
611
+ return;
612
+ const seconds = heartbeat.toSeconds();
613
+ if (seconds < MIN_WORKER_TERMINATION_HEARTBEAT.toSeconds() ||
614
+ seconds > MAX_WORKER_TERMINATION_HEARTBEAT.toSeconds()) {
615
+ throw new Error(`workerTerminationHeartbeat must be between ${MIN_WORKER_TERMINATION_HEARTBEAT.toSeconds()} ` +
616
+ `and ${MAX_WORKER_TERMINATION_HEARTBEAT.toSeconds()} seconds (got ${seconds}): shorter ` +
617
+ "cuts off the worker's 90-second drain, and Auto Scaling allows at most two hours.");
618
+ }
619
+ }
477
620
  /**
478
621
  * Core CDK construct for CanopyCMS deployment.
479
622
  *
@@ -482,7 +625,8 @@ export const DEFAULT_CMS_RESERVED_CONCURRENCY = 50;
482
625
  * - EFS filesystem with access point at /workspace
483
626
  * - Lambda function (Docker image, EFS mount, private subnet, no internet)
484
627
  * - Lambda Function URL (for CloudFront origin)
485
- * - EC2 Worker (t4g.nano spot in ASG, public subnet, EFS mount, systemd) -
628
+ * - EC2 Worker (on-demand t4g.nano by default, in an ASG, public subnet, EFS
629
+ * mount, systemd, a terminating lifecycle hook it completes once drained) -
486
630
  * rolled via the ASG's UpdatePolicy by every deploy that changes its launch
487
631
  * template, so a changed worker bundle reaches the instance instead of
488
632
  * sitting unused in a launch template until the next spot interruption (see
@@ -526,19 +670,43 @@ export class CanopyCmsService extends Construct {
526
670
  }
527
671
  // Both are interpolated into a git ref and the worker's `.env` heredoc, so
528
672
  // both are guarded at synth - see their doc comments for the failure each
529
- // prevents. `settingsBranch` stays `undefined` (not stamped at all) unless
530
- // the adopter explicitly set it: an absent env var and an empty one are NOT
531
- // the same to the worker, which falls through to a computed name only when
673
+ // prevents. Each stays `undefined` (not stamped at all) unless the adopter
674
+ // explicitly set it, so the worker detects the base branch and computes the
675
+ // settings branch name. For the settings branch an absent env var and an
676
+ // empty one are NOT the same: the worker computes the name only when
532
677
  // `CANOPYCMS_SETTINGS_BRANCH` is unset entirely.
533
- const baseBranch = assertValidGitBranchName('baseBranch', props.baseBranch ?? 'main');
678
+ const baseBranch = props.baseBranch !== undefined
679
+ ? assertValidGitBranchName('baseBranch', props.baseBranch)
680
+ : undefined;
534
681
  const settingsBranch = props.settingsBranch !== undefined
535
682
  ? assertValidGitBranchName('settingsBranch', props.settingsBranch)
536
683
  : undefined;
684
+ const protectedBranches = assertValidProtectedBranches(props.protectedBranches ?? []);
685
+ const workerCapacity = props.workerCapacity ?? { type: 'on-demand' };
686
+ const workerInstanceTypes = workerCapacity.type === 'spot'
687
+ ? (workerCapacity.instanceTypes ??
688
+ DEFAULT_SPOT_INSTANCE_TYPES.map((name) => new ec2.InstanceType(name)))
689
+ : [
690
+ workerCapacity.instanceType ??
691
+ ec2.InstanceType.of(ec2.InstanceClass.T4G, ec2.InstanceSize.NANO),
692
+ ];
693
+ if (workerInstanceTypes.length === 0) {
694
+ throw new Error('workerCapacity.instanceTypes must name at least one instance type');
695
+ }
696
+ workerInstanceTypes.forEach(assertGravitonInstanceType);
697
+ if (workerCapacity.type === 'spot' && workerCapacity.maxPrice !== undefined) {
698
+ const price = Number(workerCapacity.maxPrice);
699
+ if (!Number.isFinite(price) || price <= 0) {
700
+ throw new Error(`workerCapacity.maxPrice must be a positive USD-per-hour amount (got "${workerCapacity.maxPrice}")`);
701
+ }
702
+ }
703
+ const workerTerminationHeartbeat = props.workerTerminationHeartbeat ?? Duration.minutes(5);
704
+ assertWorkerTerminationHeartbeat(workerTerminationHeartbeat);
537
705
  // Checked at the top so a misconfigured pair fails `cdk synth` rather than
538
706
  // `cdk deploy`-then-restart-loop.
539
707
  //
540
708
  // WHICH CREDENTIAL first, then whether each is well formed: the order is
541
- // load-bearing. Reversed, an adopter following docs/adopter-migration.md who
709
+ // load-bearing. Reversed, an adopter following CHANGELOG.md who
542
710
  // removes `githubTokenSecretArn` and overlooks `githubTokenSecretJsonField`
543
711
  // is told "Set githubTokenSecretArn, or drop githubTokenSecretJsonField" --
544
712
  // pointing them back at the credential they were just told to delete, and at
@@ -621,6 +789,19 @@ export class CanopyCmsService extends Construct {
621
789
  description: 'CanopyCMS EFS',
622
790
  allowAllOutbound: false,
623
791
  });
792
+ // With any policy in effect, EFS evaluates a client that mounts without
793
+ // `iam` against the Principal "*" statements alone, so a policy granting
794
+ // "*" nothing is what requires IAM: the worker and the Lambda get their
795
+ // access from their roles' grants (below, and CDK's for the Lambda's access
796
+ // point). Lambda uses TLS for every file-system connection
797
+ // (docs.aws.amazon.com/lambda/latest/dg/security-dataprotection.html). `allowAnonymousAccess: true` keeps CDK from adding its own
798
+ // "*" ClientWrite/ClientRootAccess statement, which it does under the
799
+ // `@aws-cdk/aws-efs:denyAnonymousAccess` flag or after a `grant*` call.
800
+ const efsClientActions = [
801
+ 'elasticfilesystem:ClientMount',
802
+ 'elasticfilesystem:ClientWrite',
803
+ 'elasticfilesystem:ClientRootAccess',
804
+ ];
624
805
  this.fileSystem = new efs.FileSystem(this, 'FileSystem', {
625
806
  vpc: this.vpc,
626
807
  encrypted: true,
@@ -628,6 +809,20 @@ export class CanopyCmsService extends Construct {
628
809
  removalPolicy: props.efsRemovalPolicy ?? RemovalPolicy.RETAIN,
629
810
  vpcSubnets: { subnetType: ec2.SubnetType.PRIVATE_ISOLATED },
630
811
  securityGroup: efsSg,
812
+ enableAutomaticBackups: props.efsBackup ?? true,
813
+ allowAnonymousAccess: true,
814
+ fileSystemPolicy: (props.efsEnforceIamAndTls ?? true)
815
+ ? new iam.PolicyDocument({
816
+ statements: [
817
+ new iam.PolicyStatement({
818
+ effect: iam.Effect.DENY,
819
+ principals: [new iam.AnyPrincipal()],
820
+ actions: efsClientActions,
821
+ conditions: { Bool: { 'aws:SecureTransport': 'false' } },
822
+ }),
823
+ ],
824
+ })
825
+ : undefined,
631
826
  });
632
827
  const accessPoint = this.fileSystem.addAccessPoint('WorkspaceAP', {
633
828
  path: '/workspace',
@@ -823,8 +1018,26 @@ export class CanopyCmsService extends Construct {
823
1018
  resources: secretsArns,
824
1019
  }));
825
1020
  }
826
- // Worker needs EFS access (handled via security group, but mount needs ec2:DescribeAvailabilityZones)
827
- workerRole.addManagedPolicy(iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonElasticFileSystemClientReadWriteAccess'));
1021
+ // The worker's `iam` mount, through the shared access point only. The
1022
+ // describe calls serve amazon-efs-utils' fallback to a mount target's IP
1023
+ // address when the file system's DNS name does not resolve (it needs
1024
+ // botocore too).
1025
+ workerRole.addToPolicy(new iam.PolicyStatement({
1026
+ actions: ['elasticfilesystem:ClientMount', 'elasticfilesystem:ClientWrite'],
1027
+ resources: [this.fileSystem.fileSystemArn],
1028
+ conditions: {
1029
+ StringEquals: { 'elasticfilesystem:AccessPointArn': accessPoint.accessPointArn },
1030
+ },
1031
+ }));
1032
+ workerRole.addToPolicy(new iam.PolicyStatement({
1033
+ actions: ['elasticfilesystem:DescribeMountTargets'],
1034
+ resources: [this.fileSystem.fileSystemArn],
1035
+ }));
1036
+ workerRole.addToPolicy(new iam.PolicyStatement({
1037
+ // Supports no resource-level permissions.
1038
+ actions: ['ec2:DescribeAvailabilityZones'],
1039
+ resources: ['*'],
1040
+ }));
828
1041
  // Observation channel for a NAT-less deploy (SSM Session Manager / send-command);
829
1042
  // worker SG already allows 443 egress.
830
1043
  workerRole.addManagedPolicy(iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore'));
@@ -839,41 +1052,110 @@ export class CanopyCmsService extends Construct {
839
1052
  // CreateLogStream + PutLogEvents scoped to this group only (least privilege;
840
1053
  // the group is pre-created by CFN so the agent never needs CreateLogGroup).
841
1054
  this.workerLogGroup.grantWrite(workerRole);
842
- // The worker is bundled with esbuild into a single JS file (pnpm run build:worker)
843
- const workerAsset = new s3assets.Asset(this, 'WorkerCode', {
844
- path: path.join(__dirname, '../../worker/dist'),
845
- });
846
- workerAsset.grantRead(workerRole);
1055
+ if (props.alarmTopic) {
1056
+ // Metrics are keyed by namespace and name alone, so the name carries this
1057
+ // construct's unique id: two services in one account and region must not
1058
+ // share one count. Capped at CloudWatch's 255 characters.
1059
+ const metricPrefix = 'WorkerGitSyncCycles-';
1060
+ const syncCycles = new logs.MetricFilter(this, 'WorkerSyncCycles', {
1061
+ logGroup: this.workerLogGroup,
1062
+ filterPattern: logs.FilterPattern.literal(`"${WORKER_SYNC_LOG_PHRASE}"`),
1063
+ metricNamespace: 'CanopyCMS',
1064
+ metricName: `${metricPrefix}${Names.uniqueResourceName(this, { maxLength: 255 - metricPrefix.length })}`,
1065
+ metricValue: '1',
1066
+ });
1067
+ // Fires when 3 consecutive 10-minute periods hold no git-sync cycle line.
1068
+ // The worker logs one every cycle (default 5 minutes), even while the
1069
+ // schema gate holds the base branch. 30 minutes leaves room for a normal
1070
+ // replacement: the terminating hook's drain
1071
+ // (`workerTerminationHeartbeat`, 5 minutes by default), a boot of a few
1072
+ // minutes plus its package upgrade, and the first sync. So a deploy or a
1073
+ // spot replacement does not page, while a crash loop, a boot loop, a worker that never started and a
1074
+ // worker whose loops have stopped all go silent and do. Missing data
1075
+ // counts as a breach because a boot loop dies before the CloudWatch agent
1076
+ // is installed and so writes no events at all. The construct does not
1077
+ // expose CANOPYCMS_GIT_SYNC_INTERVAL; an adopter who sets it above about
1078
+ // 10 minutes on the instance needs a wider window.
1079
+ this.workerDownAlarm = new cloudwatch.Alarm(this, 'WorkerDownAlarm', {
1080
+ metric: syncCycles.metric({ statistic: 'Sum', period: Duration.minutes(10) }),
1081
+ threshold: 1,
1082
+ comparisonOperator: cloudwatch.ComparisonOperator.LESS_THAN_THRESHOLD,
1083
+ evaluationPeriods: 3,
1084
+ datapointsToAlarm: 3,
1085
+ treatMissingData: cloudwatch.TreatMissingData.BREACHING,
1086
+ alarmDescription: `The CMS worker has not run a git sync for 30 minutes, so publishing and branch syncing ` +
1087
+ `are stalled. Check System health in the editor's admin panel and the worker log group ` +
1088
+ `${this.workerLogGroup.logGroupName}.`,
1089
+ });
1090
+ const topicAction = new cloudwatchActions.SnsAction(props.alarmTopic);
1091
+ this.workerDownAlarm.addAlarmAction(topicAction);
1092
+ this.workerDownAlarm.addOkAction(topicAction);
1093
+ const unpatchedPrefix = 'WorkerUnpatchedBoots-';
1094
+ const unpatchedBoots = new logs.MetricFilter(this, 'WorkerUnpatchedBoots', {
1095
+ logGroup: this.workerLogGroup,
1096
+ filterPattern: logs.FilterPattern.literal(`"${WORKER_UNPATCHED_LOG_PHRASE}"`),
1097
+ metricNamespace: 'CanopyCMS',
1098
+ metricName: `${unpatchedPrefix}${Names.uniqueResourceName(this, { maxLength: 255 - unpatchedPrefix.length })}`,
1099
+ metricValue: '1',
1100
+ });
1101
+ // One line per unpatched boot, so the alarm returns to OK one period
1102
+ // later on its own; an OK notification would only repeat the alarm.
1103
+ this.workerUnpatchedAlarm = new cloudwatch.Alarm(this, 'WorkerUnpatchedAlarm', {
1104
+ metric: unpatchedBoots.metric({ statistic: 'Sum', period: Duration.minutes(10) }),
1105
+ threshold: 1,
1106
+ comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
1107
+ evaluationPeriods: 1,
1108
+ treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
1109
+ alarmDescription: `A CMS worker boot could not upgrade its packages and started on the AMI's. It is ` +
1110
+ `working, and the next boot retries the upgrade. The boot's console output ` +
1111
+ `(aws ec2 get-console-output) and the worker log group ` +
1112
+ `${this.workerLogGroup.logGroupName} name the failure.`,
1113
+ });
1114
+ this.workerUnpatchedAlarm.addAlarmAction(topicAction);
1115
+ }
1116
+ const workerBundle = workerBundleSource(this, props.workerCode ?? { source: 'asset' }, workerRole);
1117
+ this.workerBundleBucket = workerBundle.bundleBucket;
1118
+ this.workerBundleSha256Parameter = workerBundle.sha256Parameter;
847
1119
  // Name/value PAIRS rather than pre-formatted lines: every value then flows
848
1120
  // through `assertEnvSafe` in the single `map` below, so a value added here
849
1121
  // later is guarded whether or not whoever adds it remembers to.
1122
+ //
1123
+ // Each also names the process that reads it under the split. The worker
1124
+ // gets only what it needs to find the repository and the workspace; every
1125
+ // secret, every name of one (ARNs, JSON fields, the App's ids), what
1126
+ // reaches AWS, and what only the gateway enforces go to gateway.env, which
1127
+ // the worker cannot read. The single-process unit reads both files.
850
1128
  const envEntries = [
851
- ['CANOPYCMS_WORKSPACE_ROOT', EFS_MOUNT_PATH],
852
- ['CANOPYCMS_GITHUB_OWNER', props.githubOwner],
853
- ['CANOPYCMS_GITHUB_REPO', props.githubRepo],
854
- ['CANOPYCMS_BASE_BRANCH', baseBranch],
1129
+ ['CANOPYCMS_WORKSPACE_ROOT', EFS_MOUNT_PATH, 'worker'],
1130
+ ['CANOPYCMS_GITHUB_OWNER', props.githubOwner, 'both'],
1131
+ ['CANOPYCMS_GITHUB_REPO', props.githubRepo, 'both'],
855
1132
  // The SAME string the Lambda's environment gets above, including an
856
1133
  // `environment.CANOPYCMS_DEPLOYMENT_NAME` override - the two halves
857
1134
  // resolve one settings branch (`canopycms-settings-<name>`) between them,
858
1135
  // and disagreeing here is what the runtime warning detects.
859
- ['CANOPYCMS_DEPLOYMENT_NAME', deploymentName],
1136
+ ['CANOPYCMS_DEPLOYMENT_NAME', deploymentName, 'worker'],
860
1137
  // The AWS SDK JS v3 cannot resolve a region from IMDS on its own - without
861
- // this the worker's bare `SecretsManagerClient({})` crash-loops with
862
- // "Region is missing".
863
- ['AWS_REGION', Stack.of(this).region],
1138
+ // this a bare `SecretsManagerClient({})` crash-loops with "Region is
1139
+ // missing".
1140
+ ['AWS_REGION', Stack.of(this).region, 'gateway'],
864
1141
  ];
1142
+ if (baseBranch !== undefined) {
1143
+ // The worker syncs it; the gateway never pushes it.
1144
+ envEntries.push(['CANOPYCMS_BASE_BRANCH', baseBranch, 'both']);
1145
+ }
865
1146
  if (props.githubTokenSecretArn) {
866
- envEntries.push(['CANOPYCMS_GITHUB_TOKEN_SECRET_ARN', props.githubTokenSecretArn]);
1147
+ envEntries.push(['CANOPYCMS_GITHUB_TOKEN_SECRET_ARN', props.githubTokenSecretArn, 'gateway']);
867
1148
  }
868
1149
  // The JSON-field vars need NO IAM change, and that is not an oversight: a
869
1150
  // field is a key inside a secret's value, not a separately grantable
870
1151
  // resource. `secretsmanager:GetSecretValue` on the secret -- already granted
871
- // above from the same ARN prop -- returns the whole document, and the worker
1152
+ // above from the same ARN prop -- returns the whole document, and the gateway
872
1153
  // picks the field out of it in `getSecret` (worker/secrets.ts).
873
1154
  if (props.githubTokenSecretJsonField) {
874
1155
  envEntries.push([
875
1156
  'CANOPYCMS_GITHUB_TOKEN_SECRET_JSON_FIELD',
876
1157
  props.githubTokenSecretJsonField,
1158
+ 'gateway',
877
1159
  ]);
878
1160
  }
879
1161
  // GitHub App credentials, stamped individually even though
@@ -881,56 +1163,102 @@ export class CanopyCmsService extends Construct {
881
1163
  // future prop added to the App set then cannot be silently dropped by a
882
1164
  // condition that names only its siblings.
883
1165
  if (props.githubAppId) {
884
- envEntries.push(['CANOPYCMS_GITHUB_APP_ID', props.githubAppId]);
1166
+ envEntries.push(['CANOPYCMS_GITHUB_APP_ID', props.githubAppId, 'gateway']);
885
1167
  }
886
1168
  if (props.githubAppInstallationId) {
887
- envEntries.push(['CANOPYCMS_GITHUB_APP_INSTALLATION_ID', props.githubAppInstallationId]);
1169
+ envEntries.push([
1170
+ 'CANOPYCMS_GITHUB_APP_INSTALLATION_ID',
1171
+ props.githubAppInstallationId,
1172
+ 'gateway',
1173
+ ]);
888
1174
  }
889
1175
  if (props.githubAppPrivateKeySecretArn) {
890
1176
  envEntries.push([
891
1177
  'CANOPYCMS_GITHUB_APP_PRIVATE_KEY_SECRET_ARN',
892
1178
  props.githubAppPrivateKeySecretArn,
1179
+ 'gateway',
893
1180
  ]);
894
1181
  }
895
1182
  if (props.githubAppPrivateKeySecretJsonField) {
896
1183
  envEntries.push([
897
1184
  'CANOPYCMS_GITHUB_APP_PRIVATE_KEY_SECRET_JSON_FIELD',
898
1185
  props.githubAppPrivateKeySecretJsonField,
1186
+ 'gateway',
899
1187
  ]);
900
1188
  }
901
1189
  if (props.clerkSecretKeySecretArn) {
902
- envEntries.push(['CLERK_SECRET_KEY_SECRET_ARN', props.clerkSecretKeySecretArn]);
1190
+ envEntries.push(['CLERK_SECRET_KEY_SECRET_ARN', props.clerkSecretKeySecretArn, 'gateway']);
903
1191
  }
904
1192
  if (props.clerkSecretKeySecretJsonField) {
905
- envEntries.push(['CLERK_SECRET_KEY_SECRET_JSON_FIELD', props.clerkSecretKeySecretJsonField]);
1193
+ envEntries.push([
1194
+ 'CLERK_SECRET_KEY_SECRET_JSON_FIELD',
1195
+ props.clerkSecretKeySecretJsonField,
1196
+ 'gateway',
1197
+ ]);
1198
+ }
1199
+ if (workerCapacity.type === 'spot') {
1200
+ // The termination watch is the gateway's.
1201
+ envEntries.push([WORKER_CAPACITY_ENV, 'spot', 'gateway']);
906
1202
  }
907
1203
  if (settingsBranch !== undefined) {
908
1204
  // Only when explicitly set - an absent prop must keep today's behavior
909
1205
  // (the worker falls through to the computed `canopycms-settings-<name>`),
910
1206
  // so this must never stamp an empty string either way.
911
- envEntries.push(['CANOPYCMS_SETTINGS_BRANCH', settingsBranch]);
1207
+ envEntries.push(['CANOPYCMS_SETTINGS_BRANCH', settingsBranch, 'worker']);
912
1208
  }
913
- const envFileContent = envEntries
1209
+ if (protectedBranches.length > 0) {
1210
+ // Only the gateway enforces it, and a split worker refuses it.
1211
+ envEntries.push(['CANOPYCMS_PROTECTED_BRANCHES', protectedBranches.join(','), 'gateway']);
1212
+ }
1213
+ const envFile = (readers) => envEntries
1214
+ .filter(([, , reader]) => readers.includes(reader))
914
1215
  .map(([name, value]) => `${name}=${assertEnvSafe(name, value)}`)
915
1216
  .join('\n');
916
- const efsMountOptions = `tls,accesspoint=${accessPoint.accessPointId}`;
1217
+ const workerEnv = envFile(['worker', 'both']);
1218
+ const gatewayEnv = envFile(['gateway', 'both']);
1219
+ // After assertEnvSafe, whose leading-quote refusal is the more specific.
1220
+ if (settingsBranch !== undefined)
1221
+ assertGatewayBranchName('settingsBranch', settingsBranch);
1222
+ const efsMountOptions = `tls,iam,accesspoint=${accessPoint.accessPointId}`;
917
1223
  const userData = ec2.UserData.forLinux();
918
- userData.addCommands('#!/bin/bash', 'set -euo pipefail', '', '# FAIL-FAST. Every step below is required for the worker to exist at', '# all, and until this trap existed a failure in any of them aborted', '# user-data BEFORE the systemd unit was written -- leaving an instance', "# that runs, passes the ASG's EC2-only health check indefinitely, and", '# does nothing. cfn-signal is deliberately not used here (it would', '# prove nothing about READINESS, which is the argument recorded below),', '# but that argument never covered a boot-SCRIPT failure: `cdk deploy`', '# reported success while publishes queued on EFS, the auth cache went', '# stale and PRs stopped being created, until a human noticed the admin', '# panel showing the worker absent.', '#', '# Shutting down makes the instance fail its EC2 health check, so the ASG', '# replaces it -- which is the only automatic recovery available in this', '# topology. The echo lands in the console log, readable via', '# `aws ec2 get-console-output`, because the CloudWatch agent is itself', '# configured further down and cannot be relied on to exist yet.', 'trap \'echo "canopy-worker user-data FAILED at line $LINENO (exit $?)" >&2; shutdown -h now\' ERR', '', '# Bounded retry for the network-dependent steps. Package mirrors and S3', '# have transient failures; a single flake should cost seconds, not an', '# instance replacement.', 'retry() {', ' local n=0', ' until "$@"; do', ' n=$((n + 1))', ' if [ "$n" -ge 5 ]; then', ' echo "canopy-worker: command failed after $n attempts: $*" >&2', ' return 1', ' fi', ' sleep $((n * 5))', ' done', '}', '', '# Install dependencies (unzip is not guaranteed in the AL2023 AMI)', 'retry dnf install -y git unzip', "# Node comes from AL2023's own repos, NOT a piped third-party installer.", '# The previous boot curled the NodeSource RPM setup script straight into', '# bash, so every instance replacement -- which the ASG performs on every', '# `cdk deploy`, plus every spot interruption -- depended on a third party', '# being reachable. cms-deploy.test.ts asserts that URL never comes back,', '# which is why it is not spelled out here.', '#', '# nodejs22, not 20: Node 20 reached upstream EOL on 2026-04-30, and this', '# repo declares `engines.node: ">=22"` with .nvmrc `v22`, so the worker', '# was running an EOL runtime BELOW the floor its own code is tested at.', "# 22 (EOL 2027-04-30) matches .nvmrc, CI, and the transform Lambda's", '# NODEJS_22_X, so one runtime is tested everywhere.', '#', '# ExecStart uses the NAMESPACED /usr/bin/node-22 (see the systemd unit', '# below), not the bare `node`: AL2023 installs versioned binaries and', '# points `/usr/bin/node` at one of them through `alternatives`, whose', '# selection AWS documents as able to change at any time.', 'retry dnf install -y nodejs22', '', '# Mount EFS through the same access point, at the same path, as the', '# Lambda (see EFS_MOUNT_PATH). amazon-efs-utils refuses an access-point', '# mount without tls (efs_utils_common/mount_options.py).', 'retry dnf install -y amazon-efs-utils', `mkdir -p ${EFS_MOUNT_PATH}`, `mount -t efs -o ${efsMountOptions} ${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH}`, '# Persist the mount across instance reboots: user-data runs once per', '# instance, so without an fstab entry a plain reboot leaves /mnt/efs an', '# empty local dir and the worker would clone a divergent remote.git', '# onto the instance disk, invisible to the Lambda.', `echo '${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH} efs _netdev,${efsMountOptions} 0 0' >> /etc/fstab`, '', '# Download worker from CDK S3 Asset', `retry aws s3 cp s3://${workerAsset.s3BucketName}/${workerAsset.s3ObjectKey} /tmp/canopy-worker.zip`, 'mkdir -p /opt/canopy-worker', 'cd /opt/canopy-worker', 'unzip -o /tmp/canopy-worker.zip', '# The worker bundle is ESM (esbuild --format=esm). Without this marker a', '# .js file is CommonJS by default and the import statement fails.', '# Node >=22.7 auto-detects module syntax and would mask its absence, and', '# this instance now runs node-22 -- so the marker is kept BECAUSE that', '# auto-detection is a fallback we should not depend on, not because the', '# runtime here still needs it.', `echo '{"type":"module"}' > /opt/canopy-worker/package.json`, '', '# Write environment file for systemd service', `cat > /opt/canopy-worker/.env << 'ENVEOF'`, envFileContent, 'ENVEOF', '', '# Create systemd service', `cat > /etc/systemd/system/canopy-worker.service << 'SVCEOF'`, '[Unit]', 'Description=CanopyCMS Worker Daemon', 'After=network.target', '# Never run against an unmounted /mnt/efs (see the fstab note above).', `RequiresMountsFor=${EFS_MOUNT_PATH}`, '', '[Service]', 'Type=simple', 'User=ec2-user', 'WorkingDirectory=/opt/canopy-worker', '# Namespaced binary, not bare `node`: AL2023 points /usr/bin/node at', '# an installed version through `alternatives`, and AWS documents that', '# selection as able to change at any time. node-22 always means 22.', 'ExecStart=/usr/bin/node-22 index.js', 'Restart=always', 'RestartSec=5', 'TimeoutStartSec=300', '# File output, not journal: the CloudWatch agent cannot read journald,', '# so it tails this file instead (see the agent config below).', '# CAUTION: /var/log/canopy-worker must exist BEFORE first start. systemd', '# opens append: targets before it creates LogsDirectory= dirs', '# (systemd#27591), so without the pre-created dir the exec fails with', '# 209/STDOUT and Restart=always crash-loops forever. User-data runs', '# mkdir before systemctl start; LogsDirectory= is kept for ownership.', 'LogsDirectory=canopy-worker', 'StandardOutput=append:/var/log/canopy-worker/worker.log', 'StandardError=append:/var/log/canopy-worker/worker.log', 'EnvironmentFile=/opt/canopy-worker/.env', '', '[Install]', 'WantedBy=multi-user.target', 'SVCEOF', '', '# Set ownership for ec2-user', 'chown -R ec2-user:ec2-user /opt/canopy-worker', '', '# Pre-create the worker log dir (crash-loop guard, MUST precede the', '# first systemctl start): systemd opens StandardOutput=append: files', '# BEFORE it creates LogsDirectory= dirs (systemd#27591), so on a fresh', '# instance the unit would fail exec with 209/STDOUT and crash-loop', '# forever without this. The unit keeps LogsDirectory= for ownership', '# management on subsequent starts.', 'mkdir -p /var/log/canopy-worker', 'chown ec2-user:ec2-user /var/log/canopy-worker', '', '# Start worker', 'systemctl daemon-reload', 'systemctl enable canopy-worker', 'systemctl start canopy-worker', '', '# ---- CloudWatch log shipping ----', '# Placed AFTER worker start: with set -euo pipefail, a package/agent', '# failure here must not prevent the worker from running (shipping is', '# best-effort).', '#', '# DISARM THE FAIL-FAST TRAP AND errexit FIRST. Everything above this line', '# is required for the worker to exist at all, so a failure there should', '# replace the instance. Nothing below it is: the worker is already', '# running and healthy by this point. Leaving the trap armed would let a', '# package-mirror outage during the agent install shut down a perfectly', '# good worker -- and the ASG would relaunch straight into the same', '# outage, turning degraded log shipping into a replacement loop with the', '# worker down for its duration. That is strictly worse than the', '# best-effort behaviour this section has always documented.', 'trap - ERR', 'set +e', 'retry dnf install -y amazon-cloudwatch-agent logrotate', '', '# Bound on-disk growth; copytruncate keeps the fd the CW agent tails valid', '# (tiny copy->truncate loss window is acceptable for diagnostic logs).', `cat > /etc/logrotate.d/canopy-worker << 'ROTEOF'`, '/var/log/canopy-worker/worker.log {', ' size 10M', ' rotate 5', ' compress', ' copytruncate', ' missingok', ' notifempty', '}', 'ROTEOF', '# The config above only fires when logrotate actually runs; AL2023', '# presets may leave logrotate.timer disabled, and without it the size', '# cap never triggers and worker.log grows until the nano disk fills.', '# --now is idempotent if the timer is already enabled/running.', 'systemctl enable --now logrotate.timer', '', '# No retention_in_days here: CDK owns retention on the pre-created group.', `cat > /opt/aws/amazon-cloudwatch-agent/etc/canopy-worker-logs.json << 'CWEOF'`, '{', ' "logs": {', ' "logs_collected": {', ' "files": {', ' "collect_list": [', ' {', ' "file_path": "/var/log/canopy-worker/worker.log",', ` "log_group_name": "${this.workerLogGroup.logGroupName}",`, ' "log_stream_name": "{instance_id}",',
919
- // The worker prefixes every line with an ISO-8601 timestamp
920
- // (packages/canopycms/src/worker/log.ts). Parsing it here is what makes
921
- // CloudWatch show the time the WORKER emitted a line rather than the
922
- // time the agent shipped it - those diverge exactly when it matters
923
- // (agent hiccup, buffered burst, post-restart backlog).
924
- ' "timestamp_format": "%Y-%m-%dT%H:%M:%S.%f",',
925
- // The prefix ends in `Z`, so the parsed time is UTC. Without this the
926
- // agent would interpret it in the instance's local zone.
927
- ' "timezone": "UTC",',
928
- // "{timestamp_format}" reuses the pattern above as the multi-line start
929
- // marker: a line WITHOUT the timestamp prefix continues the previous
930
- // event instead of becoming its own. That is what keeps a stack trace as
931
- // one CloudWatch event, and why every writer to this file must go
932
- // through the worker log helpers.
933
- ' "multi_line_start_pattern": "{timestamp_format}"', ' }', ' ]', ' }', ' }', ' }', '}', 'CWEOF', '# fetch-config -s starts AND systemctl-enables the agent (reboot-persistent).', '/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/etc/canopy-worker-logs.json');
1224
+ const logGroupName = props.workerLogGroupName ?? `/canopycms/${Stack.of(this).stackName}/worker`;
1225
+ addUserData(userData, [
1226
+ {
1227
+ token: this.workerLogGroup.logGroupName,
1228
+ bytes: Token.isUnresolved(logGroupName)
1229
+ ? LOG_GROUP_NAME_MAX_BYTES
1230
+ : Buffer.byteLength(logGroupName),
1231
+ },
1232
+ ...envEntries
1233
+ .filter(([, value]) => Token.isUnresolved(value))
1234
+ .map(([, value]) => ({ token: value, bytes: ENV_TOKEN_MAX_BYTES })),
1235
+ ], protectedBranches.length, ...withoutShellComments('#!/bin/bash', 'set -euo pipefail', '', '# FAIL-FAST. Every step below is required for the worker to exist at', '# all, except the swap and the package upgrade, which are written as', '# conditions so that they cannot trip it. Until this trap existed, a', '# failure in any required step aborted user-data BEFORE the systemd unit was written -- leaving an instance', "# that runs, passes the ASG's EC2-only health check indefinitely, and", '# does nothing. cfn-signal is deliberately not used here (it would', '# prove nothing about READINESS, which is the argument recorded below),', '# but that argument never covered a boot-SCRIPT failure: `cdk deploy`', '# reported success while publishes queued on EFS, the auth cache went', '# stale and PRs stopped being created, until a human noticed the admin', '# panel showing the worker absent.', '#', '# Shutting down makes the instance fail its EC2 health check, so the ASG', '# replaces it -- which is the only automatic recovery available in this', '# topology. The echo lands in the console log, readable via', '# `aws ec2 get-console-output`, because the CloudWatch agent is itself', '# configured further down and cannot be relied on to exist yet.', 'trap \'echo "canopy-worker user-data FAILED at line $LINENO (exit $?)" >&2; shutdown -h now\' ERR', '', '# Bounded retry for the network-dependent steps. Package mirrors and S3', '# have transient failures; a single flake should cost seconds, not an', '# instance replacement. The first argument names the step in the message', '# an exhausted step leaves on the console.', 'retry() {', ' local step=$1 n=0', ' shift', ' until "$@"; do', ' n=$((n + 1))', ' if [ "$n" -ge 5 ]; then', ` echo "canopy-worker boot: '$step' failed after $n attempts" >&2`, ' return 1', ' fi', ' sleep $((n * 5))', ' done', '}', '', '# Swap, before the first dnf: loading the repository metadata takes more', '# memory than a t4g.nano has free after boot, and the kernel OOM-kills', '# dnf. It stays on afterwards at low swappiness, as headroom for git and', '# the CloudWatch agent rather than working memory. It is on the encrypted', '# root volume, so memory paged out of the worker is encrypted at rest.', '# fallocate is safe here: XFS, the root file system, supports', '# preallocated swap files. A failure only warns, because dnf may still fit', '# and a boot that stopped here would put the group into a replacement', '# loop. Each command carries `|| return 1` because errexit does not apply', '# inside a function called as a condition.', 'setup_swap() {', ' swapon --show=NAME --noheadings | grep -qx /swapfile && return 0', ' if [ "$(stat -c %s /swapfile 2>/dev/null)" != 1073741824 ]; then', ' # The file, plus 2 GiB for packages, logs and the GitHub mirror.', ' local avail_mib', ` avail_mib=$(df --output=avail -m / | tail -n 1 | tr -d ' ')`, ' case "$avail_mib" in ""|*[!0-9]*) return 1 ;; esac', ' if [ "$avail_mib" -lt 3072 ]; then', ' echo "canopy-worker boot: under 3 GiB free on /, so no swap file" >&2', ' return 1', ' fi', ' rm -f /swapfile || return 1', ' fallocate -l 1G /swapfile || return 1', ' fi', ' chmod 600 /swapfile || return 1', ' mkswap /swapfile || return 1', ' # swapon last, so that swap is never on when this returns 1.', " grep -qs '^/swapfile ' /etc/fstab || echo '/swapfile none swap defaults 0 0' >> /etc/fstab || return 1", ' mkdir -p /etc/sysctl.d || return 1', " echo 'vm.swappiness = 10' > /etc/sysctl.d/90-canopy-worker-swap.conf || return 1", ' sysctl -q -w vm.swappiness=10 || return 1', ' swapon /swapfile', '}', 'if ! setup_swap; then', ` echo "canopy-worker boot: 'swap' setup failed; continuing without swap" >&2`, 'fi', '', '# Patch first. AL2023 locks dnf to the repository release its AMI was', '# built from, so a relaunch from the AMI resolved at the last deploy', '# would otherwise install nothing newer than that AMI. Moving to the', '# latest release also makes it the one every install below resolves', '# from. The kernel is left out: a new one applies only at a reboot,', '# which nothing here performs, so it would cost boot time and change', '# nothing. Kernel fixes arrive with the AMI a deploy resolves.', '#', "# A failed upgrade does not stop the boot: the worker starts on the AMI's", '# packages and the next boot (by default the weekly recycle at the', '# latest) tries again. Failing here would put the group into a', '# replacement loop with no worker at all. The installs below then', '# resolve from the AMI release: 2023.7.20250428 carries all of them,', '# 2023.7.20250331 lacks nodejs22. The failure is written to worker.log', '# below, for the unpatched alarm.', 'WORKER_UNPATCHED=0', "if ! retry 'dnf upgrade' dnf upgrade --releasever=latest --exclude='kernel*' -y; then", ' WORKER_UNPATCHED=1', ` echo "canopy-worker boot: ${WORKER_UNPATCHED_LOG_PHRASE}" >&2`, 'fi', '', "retry 'dnf install git' dnf install -y git", "# Node comes from AL2023's own repos, NOT a piped third-party installer.", '# The previous boot curled the NodeSource RPM setup script straight into', '# bash, so every instance replacement -- which the ASG performs on every', '# `cdk deploy`, plus every spot interruption -- depended on a third party', '# being reachable. cms-deploy.test.ts asserts that URL never comes back,', '# which is why it is not spelled out here.', '#', '# nodejs22, not 20: Node 20 reached upstream EOL on 2026-04-30, and this', '# repo declares `engines.node: ">=22"` with .nvmrc `v22`, so the worker', '# was running an EOL runtime BELOW the floor its own code is tested at.', "# 22 (EOL 2027-04-30) matches .nvmrc, CI, and the transform Lambda's", '# NODEJS_22_X, so one runtime is tested everywhere.', '#', '# ExecStart uses the NAMESPACED /usr/bin/node-22 (see workerUnit in', '# worker-units.ts), not the bare `node`: AL2023 installs versioned binaries and', '# points `/usr/bin/node` at one of them through `alternatives`, whose', '# selection AWS documents as able to change at any time.', "retry 'dnf install nodejs22' dnf install -y nodejs22", '', '# Mount EFS through the same access point, at the same path, as the', '# Lambda (see EFS_MOUNT_PATH). amazon-efs-utils refuses an access-point', '# mount without tls (efs_utils_common/mount_options.py).', "retry 'dnf install amazon-efs-utils' dnf install -y amazon-efs-utils", `mkdir -p ${EFS_MOUNT_PATH}`, '# Retried: an `iam` mount also fetches instance-role credentials.', `retry 'EFS mount' mount -t efs -o ${efsMountOptions} ${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH}`, '# Persist the mount across instance reboots: user-data runs once per', '# instance, so without an fstab entry a plain reboot leaves /mnt/efs an', '# empty local dir and the worker would clone a divergent remote.git', '# onto the instance disk, invisible to the Lambda.', `echo '${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH} efs _netdev,${efsMountOptions} 0 0' >> /etc/fstab`, '', '# The bundle runs only if its sha256 is the one given here (synthesized,', '# or the workerCode parameter); a mismatch fails the boot under the trap', '# above.', `retry 'worker bundle download' aws s3 cp s3://${workerBundle.bucketName}/${workerBundle.objectKey} ${WORKER_BUNDLE_DOWNLOAD_PATH}`, `echo '${workerBundle.sha256} ${WORKER_BUNDLE_DOWNLOAD_PATH}' | sha256sum -c -`, '# Root-owned, and read-only to the services (ProtectSystem=strict in worker-units.ts).', `install -D -m 0644 ${WORKER_BUNDLE_DOWNLOAD_PATH} /opt/canopy-worker/index.js`, '# The worker bundle is ESM (esbuild --format=esm). Without this marker a', '# .js file is CommonJS by default and the import statement fails.', '# Node >=22.7 auto-detects module syntax and would mask its absence, and', '# this instance now runs node-22 -- so the marker is kept BECAUSE that', '# auto-detection is a fallback we should not depend on, not because the', '# runtime here still needs it.', `echo '{"type":"module"}' > /opt/canopy-worker/package.json`, '', '# The units and scripts below install from, checked as the bundle is and', '# unpacked where only root can read or write.', `retry 'worker instance files download' aws s3 cp s3://${workerBundle.instanceFiles.bucketName}/${workerBundle.instanceFiles.objectKey} ${INSTANCE_FILES_DOWNLOAD_PATH}`, `echo '${workerBundle.instanceFiles.sha256} ${INSTANCE_FILES_DOWNLOAD_PATH}' | sha256sum -c -`, `install -d -o root -g root -m 0700 ${INSTANCE_FILES_DIR}`, `(cd ${INSTANCE_FILES_DIR} && bash ${INSTANCE_FILES_DOWNLOAD_PATH})`, '', "# The worker's environment: nothing secret.", `cat > ${WORKER_INSTALL_DIR}/.env << '${ENV_HEREDOC_DELIMITER}'`, workerEnv, ENV_HEREDOC_DELIMITER, "# The secrets' names, root's alone until the split gives the gateway's", '# group read below: created with its mode before anything is written to', '# it, so it is never world-readable. The single unit reads both files.', `install -m 0600 -o root -g root /dev/null ${WORKER_INSTALL_DIR}/gateway.env`, `cat > ${WORKER_INSTALL_DIR}/gateway.env << '${ENV_HEREDOC_DELIMITER}'`, gatewayEnv, ENV_HEREDOC_DELIMITER, '', '# The units, chosen by the contract the bundle states in its first', `# lines (\`${WORKER_CONTRACT_BANNER_PREFIX}<n>\`), read after the sha256 check above,`, '# so it is as trusted as the bundle. A bundle at the split contract or', '# later gets the split: a worker with no network and no secret, and the', '# GitHub gateway holding them. An older one gets the single unit it was', '# built for: the split would cut it off from GitHub. Every bundle roll', '# replaces the instance, so each boot matches its own bundle.', 'WORKER_BUNDLE_CONTRACT=$(head -n 5 /opt/canopy-worker/index.js |', ` sed -n 's|^${WORKER_CONTRACT_BANNER_PREFIX}\\([0-9]\\{1,4\\}\\)$|\\1|p' | head -n 1 || true)`, `if [ "\${WORKER_BUNDLE_CONTRACT:-0}" -ge ${GATEWAY_SPLIT_CONTRACT} ]; then`, ' WORKER_SPLIT=1', '', ' # No core dumps on this instance. A piped core_pattern hands a crashing', " # process's memory and environment to a root helper, which journals", " # the environment where the worker's groups can read it; the gateway", ' # refuses to start unless this holds. systemd-sysctl has already run', ' # by now, so the file is applied here as well as at every boot.', ' # fb_tunnels_only_for_init_net keeps fallback tunnel devices out of', " # the worker's network namespace, which must hold only lo. -e skips a", ' # key this kernel lacks: the gateway refuses to start under a piped', ' # core_pattern, and the worker with any interface but lo.', ' mkdir -p /etc/sysctl.d', " cat > /etc/sysctl.d/99-canopy.conf << 'SYSCTLEOF'", 'kernel.core_pattern = core', 'fs.suid_dumpable = 0', 'net.core.fb_tunnels_only_for_init_net = 1', 'SYSCTLEOF', ' sysctl -q -e -p /etc/sysctl.d/99-canopy.conf', ' # PID 1 otherwise gives every service an unlimited core limit', ' # (DefaultLimitCORE=, systemd-system.conf(5)), so', ' # core_pattern=core would leave core files from any crashing daemon.', ' mkdir -p /etc/systemd/system.conf.d', " printf '[Manager]\\nDefaultLimitCORE=0\\n' > /etc/systemd/system.conf.d/canopy.conf", '', " # The gateway's user: a system account, so its uid is stable and", " # gateway.env can be its group's before the service first runs.", ` getent passwd ${GATEWAY_USER} >/dev/null ||`, ` useradd --system --no-create-home --shell /sbin/nologin ${GATEWAY_USER}`, ` chown root:${GATEWAY_USER} ${WORKER_INSTALL_DIR}/gateway.env`, ` chmod 0640 ${WORKER_INSTALL_DIR}/gateway.env`, '', ...['canopy-worker.service', 'canopy-github.service', 'canopy-github.socket'].map((unit) => ` install -m 0644 ${INSTANCE_FILES_DIR}/${unit} /etc/systemd/system/${unit}`), '', ' # The gateway log, root-owned (LOG_RULE, worker-units.ts), and readable', ' # by root alone.', ` install -d -o root -g root -m 0755 ${GATEWAY_LOG_DIR}`, ` install -o root -g root -m 0640 /dev/null ${GATEWAY_LOG}`, '', ' # canopy-inspect, for an operator on EFS: see canopyInspectScript', ' # (worker-units.ts).', ` install -m 0755 ${INSTANCE_FILES_DIR}/canopy-inspect ${CANOPY_INSPECT}`, ` install -m 0755 ${INSTANCE_FILES_DIR}/inspect-checks ${INSPECT_CHECKS}`, ` install -m 0644 ${INSTANCE_FILES_DIR}/inspect.rc ${INSPECT_RC}`, 'else', ' WORKER_SPLIT=0', ` install -m 0644 ${INSTANCE_FILES_DIR}/canopy-worker-single-process.service /etc/systemd/system/canopy-worker.service`, 'fi', '', '# The worker log, root-owned and created before the first start (LOG_RULE,', '# worker-units.ts). 0644, so it stays', '# readable without sudo.', `install -d -o root -g root -m 0755 ${WORKER_LOG_DIR}`, `install -o root -g root -m 0644 /dev/null ${WORKER_LOG}`, '', '# In the worker log format (canopycms worker/log.ts), so the CloudWatch', '# agent ships it as its own event: the agent reads a file it has no saved', '# position for from the start (its `from_beginning` defaults to true).', 'if [ "$WORKER_UNPATCHED" = 1 ]; then', ` echo "$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) ERROR canopy-worker boot: 'dnf upgrade' failed; ${WORKER_UNPATCHED_LOG_PHRASE}" >> ${WORKER_LOG}`, 'fi', '', '# Start the units; daemon-reload also reads the core-limit drop-in. The', '# gateway first, so its secret reads and first GitHub fetch start before', "# the worker's first call.", 'systemctl daemon-reload', 'if [ "$WORKER_SPLIT" = 1 ]; then', ' systemctl enable --now canopy-github.socket canopy-github.service', 'fi', 'systemctl enable canopy-worker', 'systemctl start canopy-worker', '', '# ---- CloudWatch log shipping ----', '# Placed AFTER worker start: with set -euo pipefail, a package/agent', '# failure here must not prevent the worker from running (shipping is', '# best-effort).', '#', '# DISARM THE FAIL-FAST TRAP AND errexit FIRST. Everything above this line', '# is required for the worker to exist at all, so a failure there should', '# replace the instance. Nothing below it is: the worker is already', '# running and healthy by this point. Leaving the trap armed would let a', '# package-mirror outage during the agent install shut down a perfectly', '# good worker -- and the ASG would relaunch straight into the same', '# outage, turning degraded log shipping into a replacement loop with the', '# worker down for its duration. That is strictly worse than the', '# best-effort behaviour this section has always documented.', 'trap - ERR', 'set +e', "retry 'dnf install amazon-cloudwatch-agent logrotate' dnf install -y amazon-cloudwatch-agent logrotate", '', '# Bound on-disk growth; copytruncate keeps the fd the CW agent tails valid', '# (tiny copy->truncate loss window is acceptable for diagnostic logs).', `cat > /etc/logrotate.d/canopy-worker << 'ROTEOF'`, `${WORKER_LOG} ${GATEWAY_LOG} {`, ' size 10M', ' rotate 5', ' compress', ' copytruncate', ' missingok', ' notifempty', '}', 'ROTEOF', '# The config above only fires when logrotate actually runs; AL2023', '# presets may leave logrotate.timer disabled, and without it the size', '# cap never triggers and worker.log grows until the nano disk fills.', '# --now is idempotent if the timer is already enabled/running.', 'systemctl enable --now logrotate.timer', '', '# No retention_in_days here: CDK owns retention on the pre-created group.', `cat > /opt/aws/amazon-cloudwatch-agent/etc/canopy-worker-logs.json << 'CWEOF'`, '{', ' "logs": {', ' "logs_collected": {', ' "files": {', ' "collect_list": [', ...[
1236
+ { file: WORKER_LOG, stream: '{instance_id}' },
1237
+ // The gateway's own record of what reached GitHub, which the worker
1238
+ // cannot write.
1239
+ { file: GATEWAY_LOG, stream: '{instance_id}-gateway' },
1240
+ ].flatMap(({ file, stream }, i, all) => [
1241
+ ' {',
1242
+ ` "file_path": "${file}",`,
1243
+ ` "log_group_name": "${this.workerLogGroup.logGroupName}",`,
1244
+ ` "log_stream_name": "${stream}",`,
1245
+ // Both prefix every line with an ISO-8601 timestamp
1246
+ // (packages/canopycms/src/worker/log.ts). Parsing it here is what makes
1247
+ // CloudWatch show the time the process emitted a line rather than the
1248
+ // time the agent shipped it - those diverge exactly when it matters
1249
+ // (agent hiccup, buffered burst, post-restart backlog).
1250
+ ' "timestamp_format": "%Y-%m-%dT%H:%M:%S.%f",',
1251
+ // The prefix ends in `Z`, so the parsed time is UTC. Without this the
1252
+ // agent would interpret it in the instance's local zone.
1253
+ ' "timezone": "UTC",',
1254
+ // "{timestamp_format}" reuses the pattern above as the multi-line start
1255
+ // marker: a line WITHOUT the timestamp prefix continues the previous
1256
+ // event instead of becoming its own. That is what keeps a stack trace as
1257
+ // one CloudWatch event, and why every writer to these files must go
1258
+ // through the worker log helpers.
1259
+ ' "multi_line_start_pattern": "{timestamp_format}"',
1260
+ i < all.length - 1 ? ' },' : ' }',
1261
+ ]), ' ]', ' }', ' }', ' }', '}', 'CWEOF', '# fetch-config -s starts AND systemctl-enables the agent (reboot-persistent).', '/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/etc/canopy-worker-logs.json'));
934
1262
  // Launch template (not the deprecated AutoScalingGroup instanceType/
935
1263
  // machineImage/... shorthand): that shorthand synthesizes an
936
1264
  // AWS::AutoScaling::LaunchConfiguration, which AWS accounts created after
@@ -938,24 +1266,60 @@ export class CanopyCmsService extends Construct {
938
1266
  // fresh adopter account. An explicit LaunchTemplate synthesizes
939
1267
  // AWS::EC2::LaunchTemplate instead, which every account can use.
940
1268
  const launchTemplate = new ec2.LaunchTemplate(this, 'WorkerLaunchTemplate', {
941
- instanceType: ec2.InstanceType.of(ec2.InstanceClass.T4G, ec2.InstanceSize.NANO),
1269
+ instanceType: workerInstanceTypes[0],
942
1270
  machineImage: ec2.MachineImage.latestAmazonLinux2023({
943
1271
  cpuType: ec2.AmazonLinuxCpuType.ARM_64,
944
1272
  }),
945
1273
  role: workerRole,
946
1274
  securityGroup: workerSg,
947
1275
  userData,
948
- spotOptions: {
949
- requestType: ec2.SpotRequestType.ONE_TIME, // required for ASG-managed spot
950
- maxPrice: parseFloat(props.spotMaxPrice ?? '0.0042'), // On-demand rate for t4g.nano
951
- },
1276
+ requireImdsv2: true,
1277
+ // 1, not AL2023's 2: everything that reads metadata runs on the host,
1278
+ // and a second hop only serves containers.
1279
+ httpPutResponseHopLimit: 1,
1280
+ blockDevices: [
1281
+ {
1282
+ // AL2023's root device. Its size is removed below.
1283
+ deviceName: '/dev/xvda',
1284
+ volume: ec2.BlockDeviceVolume.ebs(8, {
1285
+ encrypted: true,
1286
+ volumeType: ec2.EbsDeviceVolumeType.GP3,
1287
+ deleteOnTermination: true,
1288
+ }),
1289
+ },
1290
+ ],
952
1291
  });
1292
+ launchTemplate.node.defaultChild.addPropertyDeletionOverride('LaunchTemplateData.BlockDeviceMappings.0.Ebs.VolumeSize');
953
1293
  this.workerAsg = new autoscaling.AutoScalingGroup(this, 'WorkerAsg', {
954
1294
  vpc: this.vpc,
1295
+ // Public, deliberately: it reaches GitHub and AWS APIs through the
1296
+ // internet gateway instead of a NAT gateway that would cost more than
1297
+ // the rest of the stack. Its security group admits no inbound traffic.
955
1298
  vpcSubnets: { subnetType: ec2.SubnetType.PUBLIC },
956
- launchTemplate,
1299
+ // Spot goes through a mixed-instances policy rather than spot options on
1300
+ // the launch template, so the group can choose among several pools.
1301
+ ...(workerCapacity.type === 'spot'
1302
+ ? {
1303
+ mixedInstancesPolicy: {
1304
+ launchTemplate,
1305
+ launchTemplateOverrides: workerInstanceTypes.map((instanceType) => ({
1306
+ instanceType,
1307
+ })),
1308
+ instancesDistribution: {
1309
+ onDemandBaseCapacity: 0,
1310
+ onDemandPercentageAboveBaseCapacity: 0,
1311
+ spotAllocationStrategy: autoscaling.SpotAllocationStrategy.PRICE_CAPACITY_OPTIMIZED,
1312
+ spotMaxPrice: workerCapacity.maxPrice,
1313
+ },
1314
+ },
1315
+ capacityRebalance: workerCapacity.capacityRebalance ?? true,
1316
+ }
1317
+ : { launchTemplate }),
957
1318
  minCapacity: 1,
958
1319
  maxCapacity: 1,
1320
+ maxInstanceLifetime: props.workerMaxInstanceLifetime === null
1321
+ ? undefined
1322
+ : (props.workerMaxInstanceLifetime ?? Duration.days(7)),
959
1323
  // `healthChecks`, not the deprecated `healthCheck`/`HealthCheck.ec2({ grace })`:
960
1324
  // both synthesize the same HealthCheckType/HealthCheckGracePeriod, but the
961
1325
  // deprecated form prints a jsii warning on every synth -- in adopters'
@@ -966,10 +1330,10 @@ export class CanopyCmsService extends Construct {
966
1330
  // Without an updatePolicy, CloudFormation's default behavior for an ASG
967
1331
  // behind a changed launch template is to update the template resource
968
1332
  // and do NOTHING else - the running instance keeps its old user-data
969
- // (and therefore the old worker code: the worker bundle is a CDK S3
970
- // asset whose hash is interpolated into user-data's `aws s3 cp
971
- // s3://...`) until a spot interruption or a manual terminate happens
972
- // to replace it. `cdk deploy` would then silently deploy everything
1333
+ // (and therefore the old worker code: the bundle's key and sha256,
1334
+ // from the CDK asset or the workerCode parameter, are interpolated into
1335
+ // user data) until an interruption or a manual terminate happens to
1336
+ // replace it. `cdk deploy` would then silently deploy everything
973
1337
  // EXCEPT the worker. `rollingUpdate` makes CloudFormation actually
974
1338
  // terminate-and-relaunch the instance on every deploy that changes the
975
1339
  // launch template, so a worker code change actually reaches it.
@@ -982,10 +1346,14 @@ export class CanopyCmsService extends Construct {
982
1346
  // That outage is acceptable: the task queue and branch workspaces live on
983
1347
  // EFS, not on the instance, so the new instance picks up where the old one
984
1348
  // left off, and the Lambda's Save/Publish paths only enqueue task files
985
- // onto EFS and never talk to the worker directly. A task mid-flight at
986
- // termination is handled by orphan recovery on every task-queue cycle, not
987
- // only at worker boot (recoverOrphanedTasks in
988
- // CmsWorker.processTaskQueue(), packages/canopycms/src/worker/cms-worker.ts).
1349
+ // onto EFS and never talk to the worker directly. The old worker drains
1350
+ // first, behind the terminating lifecycle hook below. Orphan recovery on
1351
+ // every task-queue cycle (CmsWorker.processTaskQueue) covers a worker
1352
+ // that died without draining.
1353
+ //
1354
+ // Single-writer does not rest on this ordering: the worker lock on EFS
1355
+ // keeps a second worker from starting until the first has released it,
1356
+ // which a draining worker does last.
989
1357
  //
990
1358
  // `waitOnResourceSignals` therefore defaults to false here - see the
991
1359
  // no-cfn-signal note below.
@@ -1014,6 +1382,42 @@ export class CanopyCmsService extends Construct {
1014
1382
  // `mount -t efs` failure kills the whole bootstrap and the
1015
1383
  // EC2-health-checked ASG never notices.
1016
1384
  this.workerAsg.node.addDependency(this.fileSystem.mountTargetsAvailable);
1385
+ // Holds a terminating instance in Terminating:Wait until its worker has
1386
+ // drained and completed the action (worker/termination-watch.ts). CONTINUE
1387
+ // on timeout, so a hung worker delays termination by at most the heartbeat.
1388
+ //
1389
+ // A separate resource that depends on the group, so an upgrade that adds it
1390
+ // rolls the group first: the outgoing instance, whose worker predates the
1391
+ // drain, is never held for a heartbeat it cannot complete.
1392
+ this.workerAsg.addLifecycleHook('WorkerDrainHook', {
1393
+ lifecycleHookName: WORKER_DRAIN_HOOK_NAME,
1394
+ lifecycleTransition: autoscaling.LifecycleTransition.INSTANCE_TERMINATING,
1395
+ heartbeatTimeout: workerTerminationHeartbeat,
1396
+ defaultResult: autoscaling.DefaultResult.CONTINUE,
1397
+ });
1398
+ // A standalone policy rather than the role's default one: it names the
1399
+ // group, and the group depends (through the launch template) on the role.
1400
+ new iam.Policy(this, 'WorkerLifecyclePolicy', {
1401
+ roles: [workerRole],
1402
+ statements: [
1403
+ new iam.PolicyStatement({
1404
+ // Describe actions support no resource-level permissions.
1405
+ actions: ['autoscaling:DescribeAutoScalingInstances'],
1406
+ resources: ['*'],
1407
+ }),
1408
+ new iam.PolicyStatement({
1409
+ actions: ['autoscaling:CompleteLifecycleAction'],
1410
+ resources: [
1411
+ Stack.of(this).formatArn({
1412
+ service: 'autoscaling',
1413
+ resource: 'autoScalingGroup',
1414
+ resourceName: `*:autoScalingGroupName/${this.workerAsg.autoScalingGroupName}`,
1415
+ arnFormat: ArnFormat.COLON_RESOURCE_NAME,
1416
+ }),
1417
+ ],
1418
+ }),
1419
+ ],
1420
+ });
1017
1421
  }
1018
1422
  /**
1019
1423
  * Serve the editor from a CloudFront distribution you already own, such as