canopycms-cdk 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -12,6 +12,36 @@ import type { CanopyCmsAttachOptions } from './editor-routing.js';
12
12
  * @internal Exported for tests.
13
13
  */
14
14
  export declare function withoutShellComments(...lines: string[]): string[];
15
+ /**
16
+ * The branch namespace `template` names for `deploymentName`, refused at synth when the GitHub
17
+ * gateway or the policy would refuse it at boot, which would crash-loop the gateway.
18
+ *
19
+ * A local copy of `resolveBranchNamespaceTemplate` and `branchNamespaceProblem`
20
+ * (packages/canopycms/src/operating-mode/branch-namespace.ts), which this module cannot import.
21
+ * cms-deploy.test.ts holds the two to the fixtures in
22
+ * packages/canopycms/src/operating-mode/branch-namespace-fixtures.ts; add a case there when you
23
+ * change either copy. `{deployment}` is the only placeholder. `baseBranch` and `settingsBranch`
24
+ * may not lie inside the namespace, share a workspace directory with a branch in it, or be named
25
+ * after part of it, which the policy refuses at boot too.
26
+ */
27
+ export declare function resolveBranchNamespaceProp(template: string, deploymentName: string, branches?: {
28
+ baseBranch?: string;
29
+ settingsBranch?: string;
30
+ }): string;
31
+ /**
32
+ * Refuse `legacyBranches` the GitHub gateway would refuse at boot in `namespace`, which would
33
+ * crash-loop it (`legacyBranchProblem` and the `BranchPolicy` constructor,
34
+ * packages/canopycms/src/worker/branch-policy.ts): more than {@link MAX_LEGACY_BRANCHES} entries,
35
+ * a repeated entry, and any entry that is not an exact branch name held to git's rules and the
36
+ * gateway's (`assertGatewayBranchName`), so none holds a `*`. No entry may be a settings branch,
37
+ * which the gateway pushes unlisted, start with the namespace, which it already writes, or be
38
+ * named after part of it, since git then holds no branch in it. A comma separates entries in the
39
+ * env file, so none may hold one. cms-deploy.test.ts holds this copy and core's to
40
+ * packages/canopycms/src/worker/legacy-branch-fixtures.ts; add a case there when you change
41
+ * either copy.
42
+ * @internal Exported for tests.
43
+ */
44
+ export declare function assertValidLegacyBranches(entries: readonly string[], namespace: string): string[];
15
45
  /**
16
46
  * Default CMS Lambda timeout.
17
47
  *
@@ -340,6 +370,31 @@ export interface CanopyCmsServiceProps {
340
370
  * split), and validated at synth.
341
371
  */
342
372
  protectedBranches?: string[];
373
+ /**
374
+ * The prefix of every branch editors create, which the GitHub gateway confines its writes to
375
+ * (default: `canopycms/{deployment}/`). A template in which `{deployment}` is this stack's
376
+ * deployment name (`deploymentName`, including an `environment.CANOPYCMS_DEPLOYMENT_NAME`
377
+ * override: exactly the value stamped as `CANOPYCMS_DEPLOYMENT_NAME`) and the only placeholder
378
+ * allowed. It must end with `/`, and neither `baseBranch` nor `settingsBranch` may lie inside it,
379
+ * share a workspace directory with a branch in it, or be named after part of it.
380
+ * Resolved at synth, and stamped into both the CMS Lambda's environment and the gateway's
381
+ * `CANOPYCMS_BRANCH_NAMESPACE`, always, so the halves agree even when this prop is unset.
382
+ * Set the same value in `canopycms.config.ts`'s `branchNamespace` for local development; the
383
+ * environment wins. Use the same template in every stack sharing a repository, keeping
384
+ * `{deployment}` in it, so their namespaces never nest.
385
+ */
386
+ branchNamespace?: string;
387
+ /**
388
+ * Exact names of branches from before the branch namespace that editors are still working on,
389
+ * which the gateway may still push, delete and open pull requests from (default: none). Every
390
+ * other branch outside the namespace is refused, bar the settings branch. No entry may hold a
391
+ * `*` or a comma, start with the resolved `branchNamespace` or be named after part of it, or
392
+ * repeat. At most 1024 entries, and fewer when long: synth refuses any set that could take the
393
+ * worker's user data past EC2's 16 KiB. Stamped, comma-separated, into the GitHub gateway's
394
+ * `CANOPYCMS_LEGACY_BRANCHES`, and validated at synth. Remove an entry once its branch is
395
+ * merged or deleted.
396
+ */
397
+ legacyBranches?: string[];
343
398
  /**
344
399
  * Deployment name (default: 'prod'). Namespaces the settings branch
345
400
  * (`canopycms-settings-{deploymentName}`) so this stack's CMS Lambda/worker
@@ -396,8 +451,10 @@ export interface CanopyCmsServiceProps {
396
451
  * started and a worker whose loops have stopped. A deploy or a spot
397
452
  * replacement does not trip it. See `workerDownAlarm`.
398
453
  *
399
- * The topic is also notified when a worker boot could not upgrade its
400
- * packages and started on the AMI's. See `workerUnpatchedAlarm`.
454
+ * It is also notified when a live worker's git-sync cycles have all failed
455
+ * for an hour, and on recovery. See `workerSyncFailingAlarm`. And it is
456
+ * notified when a worker boot could not upgrade its packages and started on
457
+ * the AMI's. See `workerUnpatchedAlarm`.
401
458
  */
402
459
  alarmTopic?: sns.ITopic;
403
460
  /**
@@ -518,6 +575,11 @@ export declare class CanopyCmsService extends Construct {
518
575
  readonly workerLogGroup: logs.LogGroup;
519
576
  /** Alarms when the worker logs no git-sync cycle for 30 minutes. Set only with `alarmTopic`. */
520
577
  readonly workerDownAlarm?: cloudwatch.Alarm;
578
+ /**
579
+ * Alarms when the worker's git-sync cycles start but none completes for an hour. Set only with
580
+ * `alarmTopic`.
581
+ */
582
+ readonly workerSyncFailingAlarm?: cloudwatch.Alarm;
521
583
  /** Alarms when a worker boot started without its package upgrade. Set only with `alarmTopic`. */
522
584
  readonly workerUnpatchedAlarm?: cloudwatch.Alarm;
523
585
  /**
@@ -3,7 +3,7 @@ import { Annotations, ArnFormat, Duration, Names, RemovalPolicy, Stack, Token, T
3
3
  import { attachLambdaExecutionPolicies } from './lambda-execution-role.js';
4
4
  import { INSTANCE_FILES_DOWNLOAD_PATH, WORKER_BUNDLE_DOWNLOAD_PATH, workerBundleSource, } from './worker-bundle.js';
5
5
  import { attachEditorBehaviors } from './editor-routing.js';
6
- import { GATEWAY_SPLIT_CONTRACT, WORKER_CAPACITY_ENV, WORKER_CONTRACT_BANNER_PREFIX, WORKER_DRAIN_HOOK_NAME, WORKER_SYNC_LOG_PHRASE, } from './worker-lifecycle.js';
6
+ import { GATEWAY_SPLIT_CONTRACT, WORKER_CAPACITY_ENV, WORKER_CONTRACT_BANNER_PREFIX, WORKER_DRAIN_HOOK_NAME, WORKER_SYNC_DONE_LOG_PHRASE, WORKER_SYNC_LOG_PHRASE, } from './worker-lifecycle.js';
7
7
  import { CANOPY_INSPECT, EFS_MOUNT_PATH, GATEWAY_LOG, GATEWAY_LOG_DIR, GATEWAY_USER, INSPECT_CHECKS, INSPECT_RC, INSTANCE_FILES_DIR, WORKER_INSTALL_DIR, WORKER_LOG, WORKER_LOG_DIR, } from './worker-units.js';
8
8
  /**
9
9
  * Synth-time mirror of `resolveDeploymentName`'s rule in the `canopycms`
@@ -150,7 +150,7 @@ const LOG_GROUP_NAME_MAX_BYTES = 512;
150
150
  * {@link USER_DATA_TOKEN_MAX_BYTES}; an env value from a token longer than
151
151
  * {@link ENV_TOKEN_MAX_BYTES} is not caught.
152
152
  */
153
- function addUserData(userData, bounds, protectedBranchCount, ...lines) {
153
+ function addUserData(userData, bounds, counts, ...lines) {
154
154
  userData.addCommands(...lines);
155
155
  const bytesOf = new Map(bounds.flatMap(({ token, bytes }) => Tokenization.reverseString(token).tokens.map((resolvable) => [resolvable, bytes])));
156
156
  const longest = Tokenization.reverseString(userData.render())
@@ -161,9 +161,10 @@ function addUserData(userData, bounds, protectedBranchCount, ...lines) {
161
161
  const bytes = Buffer.byteLength(String(longest));
162
162
  if (bytes > USER_DATA_MAX_BYTES) {
163
163
  throw new Error(`CanopyCmsService: the worker's user data could reach ${bytes} bytes, over the ` +
164
- `${USER_DATA_MAX_BYTES} EC2 accepts; protectedBranches has ${protectedBranchCount} ` +
165
- `entries. Protect branches with fewer, wider * patterns, or shorten the other names ` +
166
- `passed in (secret ARNs and fields, branches, the repository, the worker log group).`);
164
+ `${USER_DATA_MAX_BYTES} EC2 accepts; protectedBranches has ${counts.protectedBranches} ` +
165
+ `entries and legacyBranches has ${counts.legacyBranches}. Protect branches with fewer, ` +
166
+ `wider * patterns, list fewer legacyBranches, or shorten the other names passed in ` +
167
+ `(secret ARNs and fields, branches, the repository, the worker log group).`);
167
168
  }
168
169
  }
169
170
  /**
@@ -300,6 +301,137 @@ function assertValidProtectedBranches(entries) {
300
301
  return entry;
301
302
  });
302
303
  }
304
+ /** The one placeholder a `branchNamespace` template may hold. */
305
+ const DEPLOYMENT_PLACEHOLDER = '{deployment}';
306
+ /** The `branchNamespace` template used when the prop is unset. */
307
+ const DEFAULT_BRANCH_NAMESPACE_TEMPLATE = `canopycms/${DEPLOYMENT_PLACEHOLDER}/`;
308
+ /**
309
+ * The branch namespace `template` names for `deploymentName`, refused at synth when the GitHub
310
+ * gateway or the policy would refuse it at boot, which would crash-loop the gateway.
311
+ *
312
+ * A local copy of `resolveBranchNamespaceTemplate` and `branchNamespaceProblem`
313
+ * (packages/canopycms/src/operating-mode/branch-namespace.ts), which this module cannot import.
314
+ * cms-deploy.test.ts holds the two to the fixtures in
315
+ * packages/canopycms/src/operating-mode/branch-namespace-fixtures.ts; add a case there when you
316
+ * change either copy. `{deployment}` is the only placeholder. `baseBranch` and `settingsBranch`
317
+ * may not lie inside the namespace, share a workspace directory with a branch in it, or be named
318
+ * after part of it, which the policy refuses at boot too.
319
+ */
320
+ export function resolveBranchNamespaceProp(template, deploymentName, branches = {}) {
321
+ const reject = (why) => {
322
+ throw new Error(`CanopyCmsService: invalid branchNamespace ${JSON.stringify(template)} -- ${why}.`);
323
+ };
324
+ const unknown = template.replaceAll(DEPLOYMENT_PLACEHOLDER, '').match(/\{[^}]*\}?/);
325
+ if (unknown) {
326
+ reject(`${JSON.stringify(unknown[0])} is not a placeholder; the only one is ${DEPLOYMENT_PLACEHOLDER}`);
327
+ }
328
+ const namespace = template.replaceAll(DEPLOYMENT_PLACEHOLDER, deploymentName);
329
+ const sample = `${namespace}x`;
330
+ if (!namespace.endsWith('/'))
331
+ reject('it must end with "/"');
332
+ if (!/^[\x21-\x7e]+$/.test(namespace))
333
+ reject('it is not printable ASCII without spaces');
334
+ if (/["%{}]/.test(namespace))
335
+ reject('it contains ", %, { or }');
336
+ if (/^(?:refs|heads|tags|remotes)\//i.test(namespace)) {
337
+ reject("it starts with 'refs/', 'heads/', 'tags/' or 'remotes/'");
338
+ }
339
+ if (sample.length > 250)
340
+ reject('a branch in it would be over 250 characters');
341
+ if (sample.includes('..'))
342
+ reject("it contains '..'");
343
+ if (sample.startsWith('/') || sample.includes('//'))
344
+ reject('it has an empty path component');
345
+ if (sample.startsWith('-'))
346
+ reject("a leading '-' parses as a git option");
347
+ if (/[~^:?*[\\]/.test(sample))
348
+ reject('it contains a character git forbids in a branch name');
349
+ if (sample.split('/').some((part) => part.startsWith('.') || part.endsWith('.lock'))) {
350
+ reject("a part of it starts with '.' or ends with '.lock'");
351
+ }
352
+ const sanitized = sample
353
+ .replace(/[^a-zA-Z0-9._-]/g, '-')
354
+ .replace(/-+/g, '-')
355
+ .replace(/^\.+/, '')
356
+ .replace(/(?<!\.)\.+$/, '');
357
+ const directoryPrefix = namespace
358
+ .replace(/[^a-zA-Z0-9._-]/g, '-')
359
+ .replace(/-+/g, '-')
360
+ .replace(/^\.+/, '')
361
+ .replace(/(?<!\.)\.+$/, '');
362
+ if ('canopycms-settings-'.startsWith(directoryPrefix) ||
363
+ sanitized.startsWith('canopycms-settings-')) {
364
+ reject('some branches in it would be canopycms-settings-* settings branches');
365
+ }
366
+ for (const [propName, branch] of [
367
+ ['baseBranch', branches.baseBranch],
368
+ ['settingsBranch', branches.settingsBranch],
369
+ ]) {
370
+ const sanitize = (name) => name
371
+ .replace(/[^a-zA-Z0-9._-]/g, '-')
372
+ .replace(/-+/g, '-')
373
+ .replace(/^\.+/, '')
374
+ .replace(/(?<!\.)\.+$/, '');
375
+ if (branch !== undefined &&
376
+ (branch.startsWith(namespace) || sanitize(branch).startsWith(sanitize(namespace)))) {
377
+ reject(`the ${propName} ${JSON.stringify(branch)} is inside it, or shares a workspace directory ` +
378
+ 'with a branch in it, and the namespace holds only the branches editors create');
379
+ }
380
+ if (branch !== undefined && namespace.startsWith(`${branch}/`)) {
381
+ reject(`the ${propName} ${JSON.stringify(branch)} is a branch named after part of it, so git can hold no branch in it`);
382
+ }
383
+ }
384
+ return namespace;
385
+ }
386
+ /**
387
+ * The most `legacyBranches` entries the gateway takes (`MAX_LEGACY_BRANCHES`,
388
+ * packages/canopycms/src/worker/branch-policy.ts). A local constant: this module imports nothing
389
+ * from core.
390
+ */
391
+ const MAX_LEGACY_BRANCHES = 1024;
392
+ /**
393
+ * Refuse `legacyBranches` the GitHub gateway would refuse at boot in `namespace`, which would
394
+ * crash-loop it (`legacyBranchProblem` and the `BranchPolicy` constructor,
395
+ * packages/canopycms/src/worker/branch-policy.ts): more than {@link MAX_LEGACY_BRANCHES} entries,
396
+ * a repeated entry, and any entry that is not an exact branch name held to git's rules and the
397
+ * gateway's (`assertGatewayBranchName`), so none holds a `*`. No entry may be a settings branch,
398
+ * which the gateway pushes unlisted, start with the namespace, which it already writes, or be
399
+ * named after part of it, since git then holds no branch in it. A comma separates entries in the
400
+ * env file, so none may hold one. cms-deploy.test.ts holds this copy and core's to
401
+ * packages/canopycms/src/worker/legacy-branch-fixtures.ts; add a case there when you change
402
+ * either copy.
403
+ * @internal Exported for tests.
404
+ */
405
+ export function assertValidLegacyBranches(entries, namespace) {
406
+ if (entries.length > MAX_LEGACY_BRANCHES) {
407
+ throw new Error(`CanopyCmsService: legacyBranches has ${entries.length} entries; the most it takes is ${MAX_LEGACY_BRANCHES}.`);
408
+ }
409
+ const seen = new Set();
410
+ return entries.map((entry) => {
411
+ const reject = (why) => {
412
+ throw new Error(`CanopyCmsService: invalid legacyBranches entry ${JSON.stringify(entry)} -- ${why}.`);
413
+ };
414
+ if (entry.includes(','))
415
+ reject('it contains a comma, which separates entries');
416
+ if (entry.includes('*'))
417
+ reject('an entry is an exact branch name, and * is not a wildcard here');
418
+ assertGatewayBranchName('legacyBranches entry', entry);
419
+ assertValidGitBranchName('legacyBranches entry', entry);
420
+ if (entry.startsWith('canopycms-settings-')) {
421
+ reject('it is a settings branch, which the gateway pushes without being listed');
422
+ }
423
+ if (entry.startsWith(namespace)) {
424
+ reject(`it starts with the branch namespace ${namespace}, so it is already in the namespace`);
425
+ }
426
+ if (namespace.startsWith(`${entry}/`)) {
427
+ reject(`it is a branch named after part of the branch namespace ${namespace}, so git can hold no branch in it`);
428
+ }
429
+ if (seen.has(entry))
430
+ reject('it is listed twice');
431
+ seen.add(entry);
432
+ return entry;
433
+ });
434
+ }
303
435
  /**
304
436
  * A Secrets Manager ARN carrying the ECS/CloudFormation JSON-field suffix,
305
437
  * i.e. `arn:…:secret:name-AbCdEf:MY_KEY::` rather than `arn:…:secret:name-AbCdEf`.
@@ -682,6 +814,19 @@ export class CanopyCmsService extends Construct {
682
814
  ? assertValidGitBranchName('settingsBranch', props.settingsBranch)
683
815
  : undefined;
684
816
  const protectedBranches = assertValidProtectedBranches(props.protectedBranches ?? []);
817
+ // The construct stamps the namespace into both halves; an override in `environment` would reach
818
+ // the Lambda alone and be replaced there, so it is refused rather than silently ignored.
819
+ if (props.environment?.['CANOPYCMS_BRANCH_NAMESPACE'] !== undefined) {
820
+ throw new Error('CanopyCmsService: set the branch namespace with the branchNamespace prop, not ' +
821
+ 'environment.CANOPYCMS_BRANCH_NAMESPACE: the prop reaches both the Lambda and the GitHub gateway.');
822
+ }
823
+ const branchNamespace = resolveBranchNamespaceProp(props.branchNamespace ?? DEFAULT_BRANCH_NAMESPACE_TEMPLATE, deploymentName, { baseBranch, settingsBranch });
824
+ // The gateway alone reads the list, from its own env file; `environment` reaches the Lambda.
825
+ if (props.environment?.['CANOPYCMS_LEGACY_BRANCHES'] !== undefined) {
826
+ throw new Error('CanopyCmsService: set legacy branches with the legacyBranches prop, not ' +
827
+ 'environment.CANOPYCMS_LEGACY_BRANCHES: the prop reaches the GitHub gateway, and the environment only the Lambda.');
828
+ }
829
+ const legacyBranches = assertValidLegacyBranches(props.legacyBranches ?? [], branchNamespace);
685
830
  const workerCapacity = props.workerCapacity ?? { type: 'on-demand' };
686
831
  const workerInstanceTypes = workerCapacity.type === 'spot'
687
832
  ? (workerCapacity.instanceTypes ??
@@ -926,12 +1071,14 @@ export class CanopyCmsService extends Construct {
926
1071
  // config. The fix lives in the image: Dockerfile.cms.template runs
927
1072
  // `git config --system safe.directory '*'`.
928
1073
  ...props.environment,
929
- // AFTER the spread, deliberately. Both values are already the adopter's
1074
+ // AFTER the spread, deliberately. The values are already the adopter's
930
1075
  // own choice (an `environment` override is folded into `deploymentName`
931
- // above and validated; CANOPY_MODE is restricted to 'prod'), so nothing
932
- // is taken away - the placement is what stops the Lambda and the
933
- // worker's `.env` holding different strings.
1076
+ // above and validated; `branchNamespace` is resolved and validated
1077
+ // above; CANOPY_MODE is restricted to 'prod'), so nothing is taken
1078
+ // away - the placement is what stops the Lambda and the gateway's
1079
+ // env file holding different strings.
934
1080
  CANOPYCMS_DEPLOYMENT_NAME: deploymentName,
1081
+ CANOPYCMS_BRANCH_NAMESPACE: branchNamespace,
935
1082
  CANOPY_MODE: 'prod',
936
1083
  },
937
1084
  });
@@ -1092,6 +1239,52 @@ export class CanopyCmsService extends Construct {
1092
1239
  const topicAction = new cloudwatchActions.SnsAction(props.alarmTopic);
1093
1240
  this.workerDownAlarm.addAlarmAction(topicAction);
1094
1241
  this.workerDownAlarm.addOkAction(topicAction);
1242
+ const completionsPrefix = 'WorkerGitSyncCompletions-';
1243
+ const syncCompletions = new logs.MetricFilter(this, 'WorkerSyncCompletions', {
1244
+ logGroup: this.workerLogGroup,
1245
+ filterPattern: logs.FilterPattern.literal(`"${WORKER_SYNC_DONE_LOG_PHRASE}"`),
1246
+ metricNamespace: 'CanopyCMS',
1247
+ metricName: `${completionsPrefix}${Names.uniqueResourceName(this, { maxLength: 255 - completionsPrefix.length })}`,
1248
+ metricValue: '1',
1249
+ // Every other event in the group emits 0, the cycle-start line included, so a period
1250
+ // with a start always has a completion count, and the expression below never compares
1251
+ // a missing one: a worker that has never completed a cycle still alarms.
1252
+ defaultValue: 0,
1253
+ });
1254
+ // A period with a cycle start is 1 when no cycle completed in it and 0 when one did; a
1255
+ // period with no start has no data point at all (IF drops it). CloudWatch judges the two
1256
+ // most recent real points, reaching back past such gaps within its evaluation range, and
1257
+ // `ignore` keeps the state when fewer remain. So a worker that is down neither pages here
1258
+ // (the down alarm owns that case, and one outage pages once) nor reads as recovered.
1259
+ // Every 30-minute window holds a start whenever cycles start less than 30 minutes apart,
1260
+ // which is where the down alarm takes over: cycles start the sync interval after the last
1261
+ // one ends, so a slow failing cycle or a longer interval cannot leave a gap between the
1262
+ // two. Two in a row, an hour or more, keeps a short GitHub incident from paging. The
1263
+ // schema gate's hold completes its cycle, and a drain is minutes long.
1264
+ this.workerSyncFailingAlarm = new cloudwatch.Alarm(this, 'WorkerSyncFailingAlarm', {
1265
+ metric: new cloudwatch.MathExpression({
1266
+ expression: 'IF(started, completed == 0)',
1267
+ usingMetrics: {
1268
+ started: syncCycles.metric({ statistic: 'Sum', period: Duration.minutes(30) }),
1269
+ completed: syncCompletions.metric({ statistic: 'Sum', period: Duration.minutes(30) }),
1270
+ },
1271
+ period: Duration.minutes(30),
1272
+ label: 'Periods with git-sync cycles and no completed one',
1273
+ }),
1274
+ threshold: 1,
1275
+ comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
1276
+ evaluationPeriods: 2,
1277
+ datapointsToAlarm: 2,
1278
+ treatMissingData: cloudwatch.TreatMissingData.IGNORE,
1279
+ alarmDescription: `The CMS worker is running, but none of its git syncs has completed for an hour, so ` +
1280
+ `publishing and branch syncing are stalled. Check GitHub's status first: an outage ` +
1281
+ `fails every sync, and this clears on its own when syncs succeed again. Otherwise read ` +
1282
+ `the last git sync error in System health in the editor's admin panel, and the worker ` +
1283
+ `log group ${this.workerLogGroup.logGroupName}. In workerCode parameter mode, a ` +
1284
+ `template deployed ahead of its bundle also fires this until the bundle rolls.`,
1285
+ });
1286
+ this.workerSyncFailingAlarm.addAlarmAction(topicAction);
1287
+ this.workerSyncFailingAlarm.addOkAction(topicAction);
1095
1288
  const unpatchedPrefix = 'WorkerUnpatchedBoots-';
1096
1289
  const unpatchedBoots = new logs.MetricFilter(this, 'WorkerUnpatchedBoots', {
1097
1290
  logGroup: this.workerLogGroup,
@@ -1134,8 +1327,11 @@ export class CanopyCmsService extends Construct {
1134
1327
  // The SAME string the Lambda's environment gets above, including an
1135
1328
  // `environment.CANOPYCMS_DEPLOYMENT_NAME` override - the two halves
1136
1329
  // resolve one settings branch (`canopycms-settings-<name>`) between them,
1137
- // and disagreeing here is what the runtime warning detects.
1138
- ['CANOPYCMS_DEPLOYMENT_NAME', deploymentName, 'worker'],
1330
+ // and disagreeing here is what the runtime warning detects. The gateway resolves the same
1331
+ // settings branch from it, the only settings branch it pushes.
1332
+ ['CANOPYCMS_DEPLOYMENT_NAME', deploymentName, 'both'],
1333
+ // The gateway enforces it and a split worker refuses it; the Lambda gets the same string.
1334
+ ['CANOPYCMS_BRANCH_NAMESPACE', branchNamespace, 'gateway'],
1139
1335
  // The AWS SDK JS v3 cannot resolve a region from IMDS on its own - without
1140
1336
  // this a bare `SecretsManagerClient({})` crash-loops with "Region is
1141
1337
  // missing".
@@ -1210,15 +1406,19 @@ export class CanopyCmsService extends Construct {
1210
1406
  envEntries.push([WORKER_CAPACITY_ENV, 'spot', 'gateway']);
1211
1407
  }
1212
1408
  if (settingsBranch !== undefined) {
1213
- // Only when explicitly set - an absent prop must keep today's behavior
1214
- // (the worker falls through to the computed `canopycms-settings-<name>`),
1215
- // so this must never stamp an empty string either way.
1216
- envEntries.push(['CANOPYCMS_SETTINGS_BRANCH', settingsBranch, 'worker']);
1409
+ // Only when explicitly set: unset, the worker and the gateway both compute
1410
+ // `canopycms-settings-<name>`, and an empty value would name no branch. Both read it: the
1411
+ // gateway pushes no settings branch but the one the worker resolves.
1412
+ envEntries.push(['CANOPYCMS_SETTINGS_BRANCH', settingsBranch, 'both']);
1217
1413
  }
1218
1414
  if (protectedBranches.length > 0) {
1219
1415
  // Only the gateway enforces it, and a split worker refuses it.
1220
1416
  envEntries.push(['CANOPYCMS_PROTECTED_BRANCHES', protectedBranches.join(','), 'gateway']);
1221
1417
  }
1418
+ if (legacyBranches.length > 0) {
1419
+ // Only the gateway enforces it, and a split worker refuses it.
1420
+ envEntries.push(['CANOPYCMS_LEGACY_BRANCHES', legacyBranches.join(','), 'gateway']);
1421
+ }
1222
1422
  const envFile = (readers) => envEntries
1223
1423
  .filter(([, , reader]) => readers.includes(reader))
1224
1424
  .map(([name, value]) => `${name}=${assertEnvSafe(name, value)}`)
@@ -1241,7 +1441,7 @@ export class CanopyCmsService extends Construct {
1241
1441
  ...envEntries
1242
1442
  .filter(([, value]) => Token.isUnresolved(value))
1243
1443
  .map(([, value]) => ({ token: value, bytes: ENV_TOKEN_MAX_BYTES })),
1244
- ], protectedBranches.length, ...withoutShellComments('#!/bin/bash', 'set -euo pipefail', '', '# FAIL-FAST. Every step below is required for the worker to exist at', '# all, except the swap and the package upgrade, which are written as', '# conditions so that they cannot trip it. Until this trap existed, a', '# failure in any required step aborted user-data BEFORE the systemd unit was written -- leaving an instance', "# that runs, passes the ASG's EC2-only health check indefinitely, and", '# does nothing. cfn-signal is deliberately not used here (it would', '# prove nothing about READINESS, which is the argument recorded below),', '# but that argument never covered a boot-SCRIPT failure: `cdk deploy`', '# reported success while publishes queued on EFS, the auth cache went', '# stale and PRs stopped being created, until a human noticed the admin', '# panel showing the worker absent.', '#', '# Shutting down makes the instance fail its EC2 health check, so the ASG', '# replaces it -- which is the only automatic recovery available in this', '# topology. The echo lands in the console log, readable via', '# `aws ec2 get-console-output`, because the CloudWatch agent is itself', '# configured further down and cannot be relied on to exist yet.', 'trap \'echo "canopy-worker user-data FAILED at line $LINENO (exit $?)" >&2; shutdown -h now\' ERR', '', '# Bounded retry for the network-dependent steps. Package mirrors and S3', '# have transient failures; a single flake should cost seconds, not an', '# instance replacement. The first argument names the step in the message', '# an exhausted step leaves on the console.', 'retry() {', ' local step=$1 n=0', ' shift', ' until "$@"; do', ' n=$((n + 1))', ' if [ "$n" -ge 5 ]; then', ` echo "canopy-worker boot: '$step' failed after $n attempts" >&2`, ' return 1', ' fi', ' sleep $((n * 5))', ' done', '}', '', '# Swap, before the first dnf: loading the repository metadata takes more', '# memory than a t4g.nano has free after boot, and the kernel OOM-kills', '# dnf. It stays on afterwards at low swappiness, as headroom for git and', '# the CloudWatch agent rather than working memory. It is on the encrypted', '# root volume, so memory paged out of the worker is encrypted at rest.', '# fallocate is safe here: XFS, the root file system, supports', '# preallocated swap files. A failure only warns, because dnf may still fit', '# and a boot that stopped here would put the group into a replacement', '# loop. Each command carries `|| return 1` because errexit does not apply', '# inside a function called as a condition.', 'setup_swap() {', ' swapon --show=NAME --noheadings | grep -qx /swapfile && return 0', ' if [ "$(stat -c %s /swapfile 2>/dev/null)" != 1073741824 ]; then', ' # The file, plus 2 GiB for packages, logs and the GitHub mirror.', ' local avail_mib', ` avail_mib=$(df --output=avail -m / | tail -n 1 | tr -d ' ')`, ' case "$avail_mib" in ""|*[!0-9]*) return 1 ;; esac', ' if [ "$avail_mib" -lt 3072 ]; then', ' echo "canopy-worker boot: under 3 GiB free on /, so no swap file" >&2', ' return 1', ' fi', ' rm -f /swapfile || return 1', ' fallocate -l 1G /swapfile || return 1', ' fi', ' chmod 600 /swapfile || return 1', ' mkswap /swapfile || return 1', ' # swapon last, so that swap is never on when this returns 1.', " grep -qs '^/swapfile ' /etc/fstab || echo '/swapfile none swap defaults 0 0' >> /etc/fstab || return 1", ' mkdir -p /etc/sysctl.d || return 1', " echo 'vm.swappiness = 10' > /etc/sysctl.d/90-canopy-worker-swap.conf || return 1", ' sysctl -q -w vm.swappiness=10 || return 1', ' swapon /swapfile', '}', 'if ! setup_swap; then', ` echo "canopy-worker boot: 'swap' setup failed; continuing without swap" >&2`, 'fi', '', '# Patch first. AL2023 locks dnf to the repository release its AMI was', '# built from, so a relaunch from the AMI resolved at the last deploy', '# would otherwise install nothing newer than that AMI. Moving to the', '# latest release also makes it the one every install below resolves', '# from. The kernel is left out: a new one applies only at a reboot,', '# which nothing here performs, so it would cost boot time and change', '# nothing. Kernel fixes arrive with the AMI a deploy resolves.', '#', "# A failed upgrade does not stop the boot: the worker starts on the AMI's", '# packages and the next boot (by default the weekly recycle at the', '# latest) tries again. Failing here would put the group into a', '# replacement loop with no worker at all. The installs below then', '# resolve from the AMI release: 2023.7.20250428 carries all of them,', '# 2023.7.20250331 lacks nodejs22. The failure is written to worker.log', '# below, for the unpatched alarm.', 'WORKER_UNPATCHED=0', "if ! retry 'dnf upgrade' dnf upgrade --releasever=latest --exclude='kernel*' -y; then", ' WORKER_UNPATCHED=1', ` echo "canopy-worker boot: ${WORKER_UNPATCHED_LOG_PHRASE}" >&2`, 'fi', '', "retry 'dnf install git' dnf install -y git", "# Node comes from AL2023's own repos, NOT a piped third-party installer.", '# The previous boot curled the NodeSource RPM setup script straight into', '# bash, so every instance replacement -- which the ASG performs on every', '# `cdk deploy`, plus every spot interruption -- depended on a third party', '# being reachable. cms-deploy.test.ts asserts that URL never comes back,', '# which is why it is not spelled out here.', '#', '# nodejs22, not 20: Node 20 reached upstream EOL on 2026-04-30, and this', '# repo declares `engines.node: ">=22"` with .nvmrc `v22`, so the worker', '# was running an EOL runtime BELOW the floor its own code is tested at.', "# 22 (EOL 2027-04-30) matches .nvmrc, CI, and the transform Lambda's", '# NODEJS_22_X, so one runtime is tested everywhere.', '#', '# ExecStart uses the NAMESPACED /usr/bin/node-22 (see workerUnit in', '# worker-units.ts), not the bare `node`: AL2023 installs versioned binaries and', '# points `/usr/bin/node` at one of them through `alternatives`, whose', '# selection AWS documents as able to change at any time.', "retry 'dnf install nodejs22' dnf install -y nodejs22", '', '# Mount EFS through the same access point, at the same path, as the', '# Lambda (see EFS_MOUNT_PATH). amazon-efs-utils refuses an access-point', '# mount without tls (efs_utils_common/mount_options.py).', "retry 'dnf install amazon-efs-utils' dnf install -y amazon-efs-utils", `mkdir -p ${EFS_MOUNT_PATH}`, '# Retried: an `iam` mount also fetches instance-role credentials.', `retry 'EFS mount' mount -t efs -o ${efsMountOptions} ${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH}`, '# Persist the mount across instance reboots: user-data runs once per', '# instance, so without an fstab entry a plain reboot leaves /mnt/efs an', '# empty local dir and the worker would clone a divergent remote.git', '# onto the instance disk, invisible to the Lambda.', `echo '${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH} efs _netdev,${efsMountOptions} 0 0' >> /etc/fstab`, '', '# The bundle runs only if its sha256 is the one given here (synthesized,', '# or the workerCode parameter); a mismatch fails the boot under the trap', '# above.', `retry 'worker bundle download' aws s3 cp s3://${workerBundle.bucketName}/${workerBundle.objectKey} ${WORKER_BUNDLE_DOWNLOAD_PATH}`, `echo '${workerBundle.sha256} ${WORKER_BUNDLE_DOWNLOAD_PATH}' | sha256sum -c -`, '# Root-owned, and read-only to the services (ProtectSystem=strict in worker-units.ts).', `install -D -m 0644 ${WORKER_BUNDLE_DOWNLOAD_PATH} /opt/canopy-worker/index.js`, '# The worker bundle is ESM (esbuild --format=esm). Without this marker a', '# .js file is CommonJS by default and the import statement fails.', '# Node >=22.7 auto-detects module syntax and would mask its absence, and', '# this instance now runs node-22 -- so the marker is kept BECAUSE that', '# auto-detection is a fallback we should not depend on, not because the', '# runtime here still needs it.', `echo '{"type":"module"}' > /opt/canopy-worker/package.json`, '', '# The units and scripts below install from, checked as the bundle is and', '# unpacked where only root can read or write.', `retry 'worker instance files download' aws s3 cp s3://${workerBundle.instanceFiles.bucketName}/${workerBundle.instanceFiles.objectKey} ${INSTANCE_FILES_DOWNLOAD_PATH}`, `echo '${workerBundle.instanceFiles.sha256} ${INSTANCE_FILES_DOWNLOAD_PATH}' | sha256sum -c -`, `install -d -o root -g root -m 0700 ${INSTANCE_FILES_DIR}`, `(cd ${INSTANCE_FILES_DIR} && bash ${INSTANCE_FILES_DOWNLOAD_PATH})`, '', "# The worker's environment: nothing secret.", `cat > ${WORKER_INSTALL_DIR}/.env << '${ENV_HEREDOC_DELIMITER}'`, workerEnv, ENV_HEREDOC_DELIMITER, "# The secrets' names, root's alone until the split gives the gateway's", '# group read below: created with its mode before anything is written to', '# it, so it is never world-readable. The single unit reads both files.', `install -m 0600 -o root -g root /dev/null ${WORKER_INSTALL_DIR}/gateway.env`, `cat > ${WORKER_INSTALL_DIR}/gateway.env << '${ENV_HEREDOC_DELIMITER}'`, gatewayEnv, ENV_HEREDOC_DELIMITER, '', '# The units, chosen by the contract the bundle states in its first', `# lines (\`${WORKER_CONTRACT_BANNER_PREFIX}<n>\`), read after the sha256 check above,`, '# so it is as trusted as the bundle. A bundle at the split contract or', '# later gets the split: a worker with no network and no secret, and the', '# GitHub gateway holding them. An older one gets the single unit it was', '# built for: the split would cut it off from GitHub. Every bundle roll', '# replaces the instance, so each boot matches its own bundle.', 'WORKER_BUNDLE_CONTRACT=$(head -n 5 /opt/canopy-worker/index.js |', ` sed -n 's|^${WORKER_CONTRACT_BANNER_PREFIX}\\([0-9]\\{1,4\\}\\)$|\\1|p' | head -n 1 || true)`, `if [ "\${WORKER_BUNDLE_CONTRACT:-0}" -ge ${GATEWAY_SPLIT_CONTRACT} ]; then`, ' WORKER_SPLIT=1', '', ' # No core dumps on this instance. A piped core_pattern hands a crashing', " # process's memory and environment to a root helper, which journals", " # the environment where the worker's groups can read it; the gateway", ' # refuses to start unless this holds. systemd-sysctl has already run', ' # by now, so the file is applied here as well as at every boot.', ' # fb_tunnels_only_for_init_net keeps fallback tunnel devices out of', " # the worker's network namespace, which must hold only lo. -e skips a", ' # key this kernel lacks: the gateway refuses to start under a piped', ' # core_pattern, and the worker with any interface but lo.', ' mkdir -p /etc/sysctl.d', " cat > /etc/sysctl.d/99-canopy.conf << 'SYSCTLEOF'", 'kernel.core_pattern = core', 'fs.suid_dumpable = 0', 'net.core.fb_tunnels_only_for_init_net = 1', 'SYSCTLEOF', ' sysctl -q -e -p /etc/sysctl.d/99-canopy.conf', ' # PID 1 otherwise gives every service an unlimited core limit', ' # (DefaultLimitCORE=, systemd-system.conf(5)), so', ' # core_pattern=core would leave core files from any crashing daemon.', ' mkdir -p /etc/systemd/system.conf.d', " printf '[Manager]\\nDefaultLimitCORE=0\\n' > /etc/systemd/system.conf.d/canopy.conf", '', " # The gateway's user: a system account, so its uid is stable and", " # gateway.env can be its group's before the service first runs.", ` getent passwd ${GATEWAY_USER} >/dev/null ||`, ` useradd --system --no-create-home --shell /sbin/nologin ${GATEWAY_USER}`, ` chown root:${GATEWAY_USER} ${WORKER_INSTALL_DIR}/gateway.env`, ` chmod 0640 ${WORKER_INSTALL_DIR}/gateway.env`, '', ...['canopy-worker.service', 'canopy-github.service', 'canopy-github.socket'].map((unit) => ` install -m 0644 ${INSTANCE_FILES_DIR}/${unit} /etc/systemd/system/${unit}`), '', ' # The gateway log, root-owned (LOG_RULE, worker-units.ts), and readable', ' # by root alone.', ` install -d -o root -g root -m 0755 ${GATEWAY_LOG_DIR}`, ` install -o root -g root -m 0640 /dev/null ${GATEWAY_LOG}`, '', ' # canopy-inspect, for an operator on EFS: see canopyInspectScript', ' # (worker-units.ts).', ` install -m 0755 ${INSTANCE_FILES_DIR}/canopy-inspect ${CANOPY_INSPECT}`, ` install -m 0755 ${INSTANCE_FILES_DIR}/inspect-checks ${INSPECT_CHECKS}`, ` install -m 0644 ${INSTANCE_FILES_DIR}/inspect.rc ${INSPECT_RC}`, 'else', ' WORKER_SPLIT=0', ` install -m 0644 ${INSTANCE_FILES_DIR}/canopy-worker-single-process.service /etc/systemd/system/canopy-worker.service`, 'fi', '', '# The worker log, root-owned and created before the first start (LOG_RULE,', '# worker-units.ts). 0644, so it stays', '# readable without sudo.', `install -d -o root -g root -m 0755 ${WORKER_LOG_DIR}`, `install -o root -g root -m 0644 /dev/null ${WORKER_LOG}`, '', '# In the worker log format (canopycms worker/log.ts), so the CloudWatch', '# agent ships it as its own event: the agent reads a file it has no saved', '# position for from the start (its `from_beginning` defaults to true).', 'if [ "$WORKER_UNPATCHED" = 1 ]; then', ` echo "$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) ERROR canopy-worker boot: 'dnf upgrade' failed; ${WORKER_UNPATCHED_LOG_PHRASE}" >> ${WORKER_LOG}`, 'fi', '', '# Start the units; daemon-reload also reads the core-limit drop-in. The', '# gateway first, so its secret reads and first GitHub fetch start before', "# the worker's first call.", 'systemctl daemon-reload', 'if [ "$WORKER_SPLIT" = 1 ]; then', ' systemctl enable --now canopy-github.socket canopy-github.service', 'fi', 'systemctl enable canopy-worker', 'systemctl start canopy-worker', '', '# ---- CloudWatch log shipping ----', '# Placed AFTER worker start: with set -euo pipefail, a package/agent', '# failure here must not prevent the worker from running (shipping is', '# best-effort).', '#', '# DISARM THE FAIL-FAST TRAP AND errexit FIRST. Everything above this line', '# is required for the worker to exist at all, so a failure there should', '# replace the instance. Nothing below it is: the worker is already', '# running and healthy by this point. Leaving the trap armed would let a', '# package-mirror outage during the agent install shut down a perfectly', '# good worker -- and the ASG would relaunch straight into the same', '# outage, turning degraded log shipping into a replacement loop with the', '# worker down for its duration. That is strictly worse than the', '# best-effort behaviour this section has always documented.', 'trap - ERR', 'set +e', "retry 'dnf install amazon-cloudwatch-agent logrotate' dnf install -y amazon-cloudwatch-agent logrotate", '', '# Bound on-disk growth; copytruncate keeps the fd the CW agent tails valid', '# (tiny copy->truncate loss window is acceptable for diagnostic logs).', `cat > /etc/logrotate.d/canopy-worker << 'ROTEOF'`, `${WORKER_LOG} ${GATEWAY_LOG} {`, ' size 10M', ' rotate 5', ' compress', ' copytruncate', ' missingok', ' notifempty', '}', 'ROTEOF', '# The config above only fires when logrotate actually runs; AL2023', '# presets may leave logrotate.timer disabled, and without it the size', '# cap never triggers and worker.log grows until the nano disk fills.', '# --now is idempotent if the timer is already enabled/running.', 'systemctl enable --now logrotate.timer', '', '# No retention_in_days here: CDK owns retention on the pre-created group.', `cat > /opt/aws/amazon-cloudwatch-agent/etc/canopy-worker-logs.json << 'CWEOF'`, '{', ' "logs": {', ' "logs_collected": {', ' "files": {', ' "collect_list": [', ...[
1444
+ ], { protectedBranches: protectedBranches.length, legacyBranches: legacyBranches.length }, ...withoutShellComments('#!/bin/bash', 'set -euo pipefail', '', '# FAIL-FAST. Every step below is required for the worker to exist at', '# all, except the swap and the package upgrade, which are written as', '# conditions so that they cannot trip it. Until this trap existed, a', '# failure in any required step aborted user-data BEFORE the systemd unit was written -- leaving an instance', "# that runs, passes the ASG's EC2-only health check indefinitely, and", '# does nothing. cfn-signal is deliberately not used here (it would', '# prove nothing about READINESS, which is the argument recorded below),', '# but that argument never covered a boot-SCRIPT failure: `cdk deploy`', '# reported success while publishes queued on EFS, the auth cache went', '# stale and PRs stopped being created, until a human noticed the admin', '# panel showing the worker absent.', '#', '# Shutting down makes the instance fail its EC2 health check, so the ASG', '# replaces it -- which is the only automatic recovery available in this', '# topology. The echo lands in the console log, readable via', '# `aws ec2 get-console-output`, because the CloudWatch agent is itself', '# configured further down and cannot be relied on to exist yet.', 'trap \'echo "canopy-worker user-data FAILED at line $LINENO (exit $?)" >&2; shutdown -h now\' ERR', '', '# Bounded retry for the network-dependent steps. Package mirrors and S3', '# have transient failures; a single flake should cost seconds, not an', '# instance replacement. The first argument names the step in the message', '# an exhausted step leaves on the console.', 'retry() {', ' local step=$1 n=0', ' shift', ' until "$@"; do', ' n=$((n + 1))', ' if [ "$n" -ge 5 ]; then', ` echo "canopy-worker boot: '$step' failed after $n attempts" >&2`, ' return 1', ' fi', ' sleep $((n * 5))', ' done', '}', '', '# Swap, before the first dnf: loading the repository metadata takes more', '# memory than a t4g.nano has free after boot, and the kernel OOM-kills', '# dnf. It stays on afterwards at low swappiness, as headroom for git and', '# the CloudWatch agent rather than working memory. It is on the encrypted', '# root volume, so memory paged out of the worker is encrypted at rest.', '# fallocate is safe here: XFS, the root file system, supports', '# preallocated swap files. A failure only warns, because dnf may still fit', '# and a boot that stopped here would put the group into a replacement', '# loop. Each command carries `|| return 1` because errexit does not apply', '# inside a function called as a condition.', 'setup_swap() {', ' swapon --show=NAME --noheadings | grep -qx /swapfile && return 0', ' if [ "$(stat -c %s /swapfile 2>/dev/null)" != 1073741824 ]; then', ' # The file, plus 2 GiB for packages, logs and the GitHub mirror.', ' local avail_mib', ` avail_mib=$(df --output=avail -m / | tail -n 1 | tr -d ' ')`, ' case "$avail_mib" in ""|*[!0-9]*) return 1 ;; esac', ' if [ "$avail_mib" -lt 3072 ]; then', ' echo "canopy-worker boot: under 3 GiB free on /, so no swap file" >&2', ' return 1', ' fi', ' rm -f /swapfile || return 1', ' fallocate -l 1G /swapfile || return 1', ' fi', ' chmod 600 /swapfile || return 1', ' mkswap /swapfile || return 1', ' # swapon last, so that swap is never on when this returns 1.', " grep -qs '^/swapfile ' /etc/fstab || echo '/swapfile none swap defaults 0 0' >> /etc/fstab || return 1", ' mkdir -p /etc/sysctl.d || return 1', " echo 'vm.swappiness = 10' > /etc/sysctl.d/90-canopy-worker-swap.conf || return 1", ' sysctl -q -w vm.swappiness=10 || return 1', ' swapon /swapfile', '}', 'if ! setup_swap; then', ` echo "canopy-worker boot: 'swap' setup failed; continuing without swap" >&2`, 'fi', '', '# Patch first. AL2023 locks dnf to the repository release its AMI was', '# built from, so a relaunch from the AMI resolved at the last deploy', '# would otherwise install nothing newer than that AMI. Moving to the', '# latest release also makes it the one every install below resolves', '# from. The kernel is left out: a new one applies only at a reboot,', '# which nothing here performs, so it would cost boot time and change', '# nothing. Kernel fixes arrive with the AMI a deploy resolves.', '#', "# A failed upgrade does not stop the boot: the worker starts on the AMI's", '# packages and the next boot (by default the weekly recycle at the', '# latest) tries again. Failing here would put the group into a', '# replacement loop with no worker at all. The installs below then', '# resolve from the AMI release: 2023.7.20250428 carries all of them,', '# 2023.7.20250331 lacks nodejs22. The failure is written to worker.log', '# below, for the unpatched alarm.', 'WORKER_UNPATCHED=0', "if ! retry 'dnf upgrade' dnf upgrade --releasever=latest --exclude='kernel*' -y; then", ' WORKER_UNPATCHED=1', ` echo "canopy-worker boot: ${WORKER_UNPATCHED_LOG_PHRASE}" >&2`, 'fi', '', "retry 'dnf install git' dnf install -y git", "# Node comes from AL2023's own repos, NOT a piped third-party installer.", '# The previous boot curled the NodeSource RPM setup script straight into', '# bash, so every instance replacement -- which the ASG performs on every', '# `cdk deploy`, plus every spot interruption -- depended on a third party', '# being reachable. cms-deploy.test.ts asserts that URL never comes back,', '# which is why it is not spelled out here.', '#', '# nodejs22, not 20: Node 20 reached upstream EOL on 2026-04-30, and this', '# repo declares `engines.node: ">=22"` with .nvmrc `v22`, so the worker', '# was running an EOL runtime BELOW the floor its own code is tested at.', "# 22 (EOL 2027-04-30) matches .nvmrc, CI, and the transform Lambda's", '# NODEJS_22_X, so one runtime is tested everywhere.', '#', '# ExecStart uses the NAMESPACED /usr/bin/node-22 (see workerUnit in', '# worker-units.ts), not the bare `node`: AL2023 installs versioned binaries and', '# points `/usr/bin/node` at one of them through `alternatives`, whose', '# selection AWS documents as able to change at any time.', "retry 'dnf install nodejs22' dnf install -y nodejs22", '', '# Mount EFS through the same access point, at the same path, as the', '# Lambda (see EFS_MOUNT_PATH). amazon-efs-utils refuses an access-point', '# mount without tls (efs_utils_common/mount_options.py).', "retry 'dnf install amazon-efs-utils' dnf install -y amazon-efs-utils", `mkdir -p ${EFS_MOUNT_PATH}`, '# Retried: an `iam` mount also fetches instance-role credentials.', `retry 'EFS mount' mount -t efs -o ${efsMountOptions} ${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH}`, '# Persist the mount across instance reboots: user-data runs once per', '# instance, so without an fstab entry a plain reboot leaves /mnt/efs an', '# empty local dir and the worker would clone a divergent remote.git', '# onto the instance disk, invisible to the Lambda.', `echo '${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH} efs _netdev,${efsMountOptions} 0 0' >> /etc/fstab`, '', '# The bundle runs only if its sha256 is the one given here (synthesized,', '# or the workerCode parameter); a mismatch fails the boot under the trap', '# above.', `retry 'worker bundle download' aws s3 cp s3://${workerBundle.bucketName}/${workerBundle.objectKey} ${WORKER_BUNDLE_DOWNLOAD_PATH}`, `echo '${workerBundle.sha256} ${WORKER_BUNDLE_DOWNLOAD_PATH}' | sha256sum -c -`, '# Root-owned, and read-only to the services (ProtectSystem=strict in worker-units.ts).', `install -D -m 0644 ${WORKER_BUNDLE_DOWNLOAD_PATH} /opt/canopy-worker/index.js`, '# The worker bundle is ESM (esbuild --format=esm). Without this marker a', '# .js file is CommonJS by default and the import statement fails.', '# Node >=22.7 auto-detects module syntax and would mask its absence, and', '# this instance now runs node-22 -- so the marker is kept BECAUSE that', '# auto-detection is a fallback we should not depend on, not because the', '# runtime here still needs it.', `echo '{"type":"module"}' > /opt/canopy-worker/package.json`, '', '# The units and scripts below install from, checked as the bundle is and', '# unpacked where only root can read or write.', `retry 'worker instance files download' aws s3 cp s3://${workerBundle.instanceFiles.bucketName}/${workerBundle.instanceFiles.objectKey} ${INSTANCE_FILES_DOWNLOAD_PATH}`, `echo '${workerBundle.instanceFiles.sha256} ${INSTANCE_FILES_DOWNLOAD_PATH}' | sha256sum -c -`, `install -d -o root -g root -m 0700 ${INSTANCE_FILES_DIR}`, `(cd ${INSTANCE_FILES_DIR} && bash ${INSTANCE_FILES_DOWNLOAD_PATH})`, '', "# The worker's environment: nothing secret.", `cat > ${WORKER_INSTALL_DIR}/.env << '${ENV_HEREDOC_DELIMITER}'`, workerEnv, ENV_HEREDOC_DELIMITER, "# The secrets' names, root's alone until the split gives the gateway's", '# group read below: created with its mode before anything is written to', '# it, so it is never world-readable. The single unit reads both files.', `install -m 0600 -o root -g root /dev/null ${WORKER_INSTALL_DIR}/gateway.env`, `cat > ${WORKER_INSTALL_DIR}/gateway.env << '${ENV_HEREDOC_DELIMITER}'`, gatewayEnv, ENV_HEREDOC_DELIMITER, '', '# The units, chosen by the contract the bundle states in its first', `# lines (\`${WORKER_CONTRACT_BANNER_PREFIX}<n>\`), read after the sha256 check above,`, '# so it is as trusted as the bundle. A bundle at the split contract or', '# later gets the split: a worker with no network and no secret, and the', '# GitHub gateway holding them. An older one gets the single unit it was', '# built for: the split would cut it off from GitHub. Every bundle roll', '# replaces the instance, so each boot matches its own bundle.', 'WORKER_BUNDLE_CONTRACT=$(head -n 5 /opt/canopy-worker/index.js |', ` sed -n 's|^${WORKER_CONTRACT_BANNER_PREFIX}\\([0-9]\\{1,4\\}\\)$|\\1|p' | head -n 1 || true)`, `if [ "\${WORKER_BUNDLE_CONTRACT:-0}" -ge ${GATEWAY_SPLIT_CONTRACT} ]; then`, ' WORKER_SPLIT=1', '', ' # No core dumps on this instance. A piped core_pattern hands a crashing', " # process's memory and environment to a root helper, which journals", " # the environment where the worker's groups can read it; the gateway", ' # refuses to start unless this holds. systemd-sysctl has already run', ' # by now, so the file is applied here as well as at every boot.', ' # fb_tunnels_only_for_init_net keeps fallback tunnel devices out of', " # the worker's network namespace, which must hold only lo. -e skips a", ' # key this kernel lacks: the gateway refuses to start under a piped', ' # core_pattern, and the worker with any interface but lo.', ' mkdir -p /etc/sysctl.d', " cat > /etc/sysctl.d/99-canopy.conf << 'SYSCTLEOF'", 'kernel.core_pattern = core', 'fs.suid_dumpable = 0', 'net.core.fb_tunnels_only_for_init_net = 1', 'SYSCTLEOF', ' sysctl -q -e -p /etc/sysctl.d/99-canopy.conf', ' # PID 1 otherwise gives every service an unlimited core limit', ' # (DefaultLimitCORE=, systemd-system.conf(5)), so', ' # core_pattern=core would leave core files from any crashing daemon.', ' mkdir -p /etc/systemd/system.conf.d', " printf '[Manager]\\nDefaultLimitCORE=0\\n' > /etc/systemd/system.conf.d/canopy.conf", '', " # The gateway's user: a system account, so its uid is stable and", " # gateway.env can be its group's before the service first runs.", ` getent passwd ${GATEWAY_USER} >/dev/null ||`, ` useradd --system --no-create-home --shell /sbin/nologin ${GATEWAY_USER}`, ` chown root:${GATEWAY_USER} ${WORKER_INSTALL_DIR}/gateway.env`, ` chmod 0640 ${WORKER_INSTALL_DIR}/gateway.env`, '', ...['canopy-worker.service', 'canopy-github.service', 'canopy-github.socket'].map((unit) => ` install -m 0644 ${INSTANCE_FILES_DIR}/${unit} /etc/systemd/system/${unit}`), '', ' # The gateway log, root-owned (LOG_RULE, worker-units.ts), and readable', ' # by root alone.', ` install -d -o root -g root -m 0755 ${GATEWAY_LOG_DIR}`, ` install -o root -g root -m 0640 /dev/null ${GATEWAY_LOG}`, '', ' # canopy-inspect, for an operator on EFS: see canopyInspectScript', ' # (worker-units.ts).', ` install -m 0755 ${INSTANCE_FILES_DIR}/canopy-inspect ${CANOPY_INSPECT}`, ` install -m 0755 ${INSTANCE_FILES_DIR}/inspect-checks ${INSPECT_CHECKS}`, ` install -m 0644 ${INSTANCE_FILES_DIR}/inspect.rc ${INSPECT_RC}`, 'else', ' WORKER_SPLIT=0', ` install -m 0644 ${INSTANCE_FILES_DIR}/canopy-worker-single-process.service /etc/systemd/system/canopy-worker.service`, 'fi', '', '# The worker log, root-owned and created before the first start (LOG_RULE,', '# worker-units.ts). 0644, so it stays', '# readable without sudo.', `install -d -o root -g root -m 0755 ${WORKER_LOG_DIR}`, `install -o root -g root -m 0644 /dev/null ${WORKER_LOG}`, '', '# In the worker log format (canopycms worker/log.ts), so the CloudWatch', '# agent ships it as its own event: the agent reads a file it has no saved', '# position for from the start (its `from_beginning` defaults to true).', 'if [ "$WORKER_UNPATCHED" = 1 ]; then', ` echo "$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) ERROR canopy-worker boot: 'dnf upgrade' failed; ${WORKER_UNPATCHED_LOG_PHRASE}" >> ${WORKER_LOG}`, 'fi', '', '# Start the units; daemon-reload also reads the core-limit drop-in. The', '# gateway first, so its secret reads and first GitHub fetch start before', "# the worker's first call.", 'systemctl daemon-reload', 'if [ "$WORKER_SPLIT" = 1 ]; then', ' systemctl enable --now canopy-github.socket canopy-github.service', 'fi', 'systemctl enable canopy-worker', 'systemctl start canopy-worker', '', '# ---- CloudWatch log shipping ----', '# Placed AFTER worker start: with set -euo pipefail, a package/agent', '# failure here must not prevent the worker from running (shipping is', '# best-effort).', '#', '# DISARM THE FAIL-FAST TRAP AND errexit FIRST. Everything above this line', '# is required for the worker to exist at all, so a failure there should', '# replace the instance. Nothing below it is: the worker is already', '# running and healthy by this point. Leaving the trap armed would let a', '# package-mirror outage during the agent install shut down a perfectly', '# good worker -- and the ASG would relaunch straight into the same', '# outage, turning degraded log shipping into a replacement loop with the', '# worker down for its duration. That is strictly worse than the', '# best-effort behaviour this section has always documented.', 'trap - ERR', 'set +e', "retry 'dnf install amazon-cloudwatch-agent logrotate' dnf install -y amazon-cloudwatch-agent logrotate", '', '# Bound on-disk growth; copytruncate keeps the fd the CW agent tails valid', '# (tiny copy->truncate loss window is acceptable for diagnostic logs).', `cat > /etc/logrotate.d/canopy-worker << 'ROTEOF'`, `${WORKER_LOG} ${GATEWAY_LOG} {`, ' size 10M', ' rotate 5', ' compress', ' copytruncate', ' missingok', ' notifempty', '}', 'ROTEOF', '# The config above only fires when logrotate actually runs; AL2023', '# presets may leave logrotate.timer disabled, and without it the size', '# cap never triggers and worker.log grows until the nano disk fills.', '# --now is idempotent if the timer is already enabled/running.', 'systemctl enable --now logrotate.timer', '', '# No retention_in_days here: CDK owns retention on the pre-created group.', `cat > /opt/aws/amazon-cloudwatch-agent/etc/canopy-worker-logs.json << 'CWEOF'`, '{', ' "logs": {', ' "logs_collected": {', ' "files": {', ' "collect_list": [', ...[
1245
1445
  { file: WORKER_LOG, stream: '{instance_id}' },
1246
1446
  // The gateway's own record of what reached GitHub, which the worker
1247
1447
  // cannot write.
@@ -48,7 +48,7 @@ export declare const WORKER_CONTRACT_BANNER_PREFIX = "// canopy-worker-contract:
48
48
  * bundle as `worker/dist/index.js.contract`, so a change-set gate can compare
49
49
  * the two before rolling a bundle. See {@link WORKER_CONTRACT_REQUIREMENTS}.
50
50
  */
51
- export declare const WORKER_CONTRACT_VERSION: 2;
51
+ export declare const WORKER_CONTRACT_VERSION: 3;
52
52
  /** The unit's `Environment=` variable carrying its worker contract version. */
53
53
  export declare const WORKER_CONTRACT_ENV = "CANOPYCMS_WORKER_CONTRACT";
54
54
  /**
@@ -68,3 +68,9 @@ export declare function workerContractShortfall(env: Readonly<Record<string, str
68
68
  * it, so the alarm misfires if the worker's text drifts from this one.
69
69
  */
70
70
  export declare const WORKER_SYNC_LOG_PHRASE = "Syncing git...";
71
+ /**
72
+ * The text of the line `syncGit()` logs when a git-sync cycle has run to its end. The
73
+ * sync-failing alarm alarms when cycles start ({@link WORKER_SYNC_LOG_PHRASE}) and none logs
74
+ * this, so it misfires if the worker's text drifts from this one.
75
+ */
76
+ export declare const WORKER_SYNC_DONE_LOG_PHRASE = "Git sync complete";
@@ -63,6 +63,7 @@ export const WORKER_CONTRACT_BANNER_PREFIX = '// canopy-worker-contract: ';
63
63
  const WORKER_CONTRACT_REQUIREMENTS = [
64
64
  'StateDirectory=canopy-worker',
65
65
  `canopy-github.socket and canopy-github.service, and ${GITHUB_GATEWAY_SOCKET_ENV} on the worker unit`,
66
+ 'CANOPYCMS_DEPLOYMENT_NAME, and CANOPYCMS_SETTINGS_BRANCH when set, in gateway.env',
66
67
  ];
67
68
  /**
68
69
  * The worker contract version: stamped into the unit by the construct (and,
@@ -109,3 +110,9 @@ export function workerContractShortfall(env) {
109
110
  * it, so the alarm misfires if the worker's text drifts from this one.
110
111
  */
111
112
  export const WORKER_SYNC_LOG_PHRASE = 'Syncing git...';
113
+ /**
114
+ * The text of the line `syncGit()` logs when a git-sync cycle has run to its end. The
115
+ * sync-failing alarm alarms when cycles start ({@link WORKER_SYNC_LOG_PHRASE}) and none logs
116
+ * this, so it misfires if the worker's text drifts from this one.
117
+ */
118
+ export const WORKER_SYNC_DONE_LOG_PHRASE = 'Git sync complete';
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "canopycms-cdk",
3
- "version": "0.7.0",
3
+ "version": "0.8.0",
4
4
  "description": "AWS CDK constructs and EC2 worker for CanopyCMS deployment",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -34,7 +34,7 @@
34
34
  "peerDependencies": {
35
35
  "aws-cdk-lib": "^2.260.0",
36
36
  "constructs": "^10.0.0",
37
- "canopycms": "0.7.0"
37
+ "canopycms": "0.8.0"
38
38
  },
39
39
  "peerDependenciesMeta": {
40
40
  "aws-cdk-lib": {
@@ -60,8 +60,8 @@
60
60
  "tsx": "^4.19.0",
61
61
  "typescript": "^5.6.3",
62
62
  "vitest": "^4.1.0",
63
- "canopycms": "0.7.0",
64
- "canopycms-auth-clerk": "0.7.0"
63
+ "canopycms": "0.8.0",
64
+ "canopycms-auth-clerk": "0.8.0"
65
65
  },
66
66
  "scripts": {
67
67
  "build": "rm -rf dist && tsc && node ../../scripts/add-js-extensions.mjs dist",
@@ -45,7 +45,7 @@ StateDirectoryMode=0700
45
45
  # 0640 root:canopy-github: the secrets, unreadable to the worker.
46
46
  EnvironmentFile=/opt/canopy-worker/gateway.env
47
47
  # What this unit provides; a bundle needing more refuses to start.
48
- Environment=CANOPYCMS_WORKER_CONTRACT=2
48
+ Environment=CANOPYCMS_WORKER_CONTRACT=3
49
49
  # Sandbox. Writable: the state dir and a private /tmp. No view of EFS,
50
50
  # which the gateway checks at boot, as it checks that cores are not
51
51
  # piped to a helper and that its core limit is zero.
@@ -45,7 +45,7 @@ StandardError=append:/var/log/canopy-worker/worker.log
45
45
  StateDirectory=canopy-worker
46
46
  EnvironmentFile=/opt/canopy-worker/.env
47
47
  # What this unit provides; a bundle needing more refuses to start.
48
- Environment=CANOPYCMS_WORKER_CONTRACT=2
48
+ Environment=CANOPYCMS_WORKER_CONTRACT=3
49
49
  # The GitHub gateway, its only way to GitHub, the Clerk API and the
50
50
  # instance role.
51
51
  Environment=CANOPYCMS_GITHUB_GATEWAY_SOCKET=/run/canopy-github/gateway.sock