canopycms-cdk 0.7.1 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/constructs/cms-service.d.ts +55 -0
- package/dist/constructs/cms-service.js +169 -15
- package/dist/constructs/worker-lifecycle.d.ts +1 -1
- package/dist/constructs/worker-lifecycle.js +1 -0
- package/package.json +4 -4
- package/worker/canopy-github.service +1 -1
- package/worker/canopy-worker.service +1 -1
- package/worker/dist/index.js +1674 -883
- package/worker/dist/index.js.contract +1 -1
- package/worker/dist/index.js.sha256 +1 -1
- package/worker/dist/instance-files.sh +2 -2
- package/worker/gateway.env.example +12 -0
|
@@ -12,6 +12,36 @@ import type { CanopyCmsAttachOptions } from './editor-routing.js';
|
|
|
12
12
|
* @internal Exported for tests.
|
|
13
13
|
*/
|
|
14
14
|
export declare function withoutShellComments(...lines: string[]): string[];
|
|
15
|
+
/**
|
|
16
|
+
* The branch namespace `template` names for `deploymentName`, refused at synth when the GitHub
|
|
17
|
+
* gateway or the policy would refuse it at boot, which would crash-loop the gateway.
|
|
18
|
+
*
|
|
19
|
+
* A local copy of `resolveBranchNamespaceTemplate` and `branchNamespaceProblem`
|
|
20
|
+
* (packages/canopycms/src/operating-mode/branch-namespace.ts), which this module cannot import.
|
|
21
|
+
* cms-deploy.test.ts holds the two to the fixtures in
|
|
22
|
+
* packages/canopycms/src/operating-mode/branch-namespace-fixtures.ts; add a case there when you
|
|
23
|
+
* change either copy. `{deployment}` is the only placeholder. `baseBranch` and `settingsBranch`
|
|
24
|
+
* may not lie inside the namespace, share a workspace directory with a branch in it, or be named
|
|
25
|
+
* after part of it, which the policy refuses at boot too.
|
|
26
|
+
*/
|
|
27
|
+
export declare function resolveBranchNamespaceProp(template: string, deploymentName: string, branches?: {
|
|
28
|
+
baseBranch?: string;
|
|
29
|
+
settingsBranch?: string;
|
|
30
|
+
}): string;
|
|
31
|
+
/**
|
|
32
|
+
* Refuse `legacyBranches` the GitHub gateway would refuse at boot in `namespace`, which would
|
|
33
|
+
* crash-loop it (`legacyBranchProblem` and the `BranchPolicy` constructor,
|
|
34
|
+
* packages/canopycms/src/worker/branch-policy.ts): more than {@link MAX_LEGACY_BRANCHES} entries,
|
|
35
|
+
* a repeated entry, and any entry that is not an exact branch name held to git's rules and the
|
|
36
|
+
* gateway's (`assertGatewayBranchName`), so none holds a `*`. No entry may be a settings branch,
|
|
37
|
+
* which the gateway pushes unlisted, start with the namespace, which it already writes, or be
|
|
38
|
+
* named after part of it, since git then holds no branch in it. A comma separates entries in the
|
|
39
|
+
* env file, so none may hold one. cms-deploy.test.ts holds this copy and core's to
|
|
40
|
+
* packages/canopycms/src/worker/legacy-branch-fixtures.ts; add a case there when you change
|
|
41
|
+
* either copy.
|
|
42
|
+
* @internal Exported for tests.
|
|
43
|
+
*/
|
|
44
|
+
export declare function assertValidLegacyBranches(entries: readonly string[], namespace: string): string[];
|
|
15
45
|
/**
|
|
16
46
|
* Default CMS Lambda timeout.
|
|
17
47
|
*
|
|
@@ -340,6 +370,31 @@ export interface CanopyCmsServiceProps {
|
|
|
340
370
|
* split), and validated at synth.
|
|
341
371
|
*/
|
|
342
372
|
protectedBranches?: string[];
|
|
373
|
+
/**
|
|
374
|
+
* The prefix of every branch editors create, which the GitHub gateway confines its writes to
|
|
375
|
+
* (default: `canopycms/{deployment}/`). A template in which `{deployment}` is this stack's
|
|
376
|
+
* deployment name (`deploymentName`, including an `environment.CANOPYCMS_DEPLOYMENT_NAME`
|
|
377
|
+
* override: exactly the value stamped as `CANOPYCMS_DEPLOYMENT_NAME`) and the only placeholder
|
|
378
|
+
* allowed. It must end with `/`, and neither `baseBranch` nor `settingsBranch` may lie inside it,
|
|
379
|
+
* share a workspace directory with a branch in it, or be named after part of it.
|
|
380
|
+
* Resolved at synth, and stamped into both the CMS Lambda's environment and the gateway's
|
|
381
|
+
* `CANOPYCMS_BRANCH_NAMESPACE`, always, so the halves agree even when this prop is unset.
|
|
382
|
+
* Set the same value in `canopycms.config.ts`'s `branchNamespace` for local development; the
|
|
383
|
+
* environment wins. Use the same template in every stack sharing a repository, keeping
|
|
384
|
+
* `{deployment}` in it, so their namespaces never nest.
|
|
385
|
+
*/
|
|
386
|
+
branchNamespace?: string;
|
|
387
|
+
/**
|
|
388
|
+
* Exact names of branches from before the branch namespace that editors are still working on,
|
|
389
|
+
* which the gateway may still push, delete and open pull requests from (default: none). Every
|
|
390
|
+
* other branch outside the namespace is refused, bar the settings branch. No entry may hold a
|
|
391
|
+
* `*` or a comma, start with the resolved `branchNamespace` or be named after part of it, or
|
|
392
|
+
* repeat. At most 1024 entries, and fewer when long: synth refuses any set that could take the
|
|
393
|
+
* worker's user data past EC2's 16 KiB. Stamped, comma-separated, into the GitHub gateway's
|
|
394
|
+
* `CANOPYCMS_LEGACY_BRANCHES`, and validated at synth. Remove an entry once its branch is
|
|
395
|
+
* merged or deleted.
|
|
396
|
+
*/
|
|
397
|
+
legacyBranches?: string[];
|
|
343
398
|
/**
|
|
344
399
|
* Deployment name (default: 'prod'). Namespaces the settings branch
|
|
345
400
|
* (`canopycms-settings-{deploymentName}`) so this stack's CMS Lambda/worker
|
|
@@ -150,7 +150,7 @@ const LOG_GROUP_NAME_MAX_BYTES = 512;
|
|
|
150
150
|
* {@link USER_DATA_TOKEN_MAX_BYTES}; an env value from a token longer than
|
|
151
151
|
* {@link ENV_TOKEN_MAX_BYTES} is not caught.
|
|
152
152
|
*/
|
|
153
|
-
function addUserData(userData, bounds,
|
|
153
|
+
function addUserData(userData, bounds, counts, ...lines) {
|
|
154
154
|
userData.addCommands(...lines);
|
|
155
155
|
const bytesOf = new Map(bounds.flatMap(({ token, bytes }) => Tokenization.reverseString(token).tokens.map((resolvable) => [resolvable, bytes])));
|
|
156
156
|
const longest = Tokenization.reverseString(userData.render())
|
|
@@ -161,9 +161,10 @@ function addUserData(userData, bounds, protectedBranchCount, ...lines) {
|
|
|
161
161
|
const bytes = Buffer.byteLength(String(longest));
|
|
162
162
|
if (bytes > USER_DATA_MAX_BYTES) {
|
|
163
163
|
throw new Error(`CanopyCmsService: the worker's user data could reach ${bytes} bytes, over the ` +
|
|
164
|
-
`${USER_DATA_MAX_BYTES} EC2 accepts; protectedBranches has ${
|
|
165
|
-
`entries. Protect branches with fewer,
|
|
166
|
-
`
|
|
164
|
+
`${USER_DATA_MAX_BYTES} EC2 accepts; protectedBranches has ${counts.protectedBranches} ` +
|
|
165
|
+
`entries and legacyBranches has ${counts.legacyBranches}. Protect branches with fewer, ` +
|
|
166
|
+
`wider * patterns, list fewer legacyBranches, or shorten the other names passed in ` +
|
|
167
|
+
`(secret ARNs and fields, branches, the repository, the worker log group).`);
|
|
167
168
|
}
|
|
168
169
|
}
|
|
169
170
|
/**
|
|
@@ -300,6 +301,137 @@ function assertValidProtectedBranches(entries) {
|
|
|
300
301
|
return entry;
|
|
301
302
|
});
|
|
302
303
|
}
|
|
304
|
+
/** The one placeholder a `branchNamespace` template may hold. */
|
|
305
|
+
const DEPLOYMENT_PLACEHOLDER = '{deployment}';
|
|
306
|
+
/** The `branchNamespace` template used when the prop is unset. */
|
|
307
|
+
const DEFAULT_BRANCH_NAMESPACE_TEMPLATE = `canopycms/${DEPLOYMENT_PLACEHOLDER}/`;
|
|
308
|
+
/**
|
|
309
|
+
* The branch namespace `template` names for `deploymentName`, refused at synth when the GitHub
|
|
310
|
+
* gateway or the policy would refuse it at boot, which would crash-loop the gateway.
|
|
311
|
+
*
|
|
312
|
+
* A local copy of `resolveBranchNamespaceTemplate` and `branchNamespaceProblem`
|
|
313
|
+
* (packages/canopycms/src/operating-mode/branch-namespace.ts), which this module cannot import.
|
|
314
|
+
* cms-deploy.test.ts holds the two to the fixtures in
|
|
315
|
+
* packages/canopycms/src/operating-mode/branch-namespace-fixtures.ts; add a case there when you
|
|
316
|
+
* change either copy. `{deployment}` is the only placeholder. `baseBranch` and `settingsBranch`
|
|
317
|
+
* may not lie inside the namespace, share a workspace directory with a branch in it, or be named
|
|
318
|
+
* after part of it, which the policy refuses at boot too.
|
|
319
|
+
*/
|
|
320
|
+
export function resolveBranchNamespaceProp(template, deploymentName, branches = {}) {
|
|
321
|
+
const reject = (why) => {
|
|
322
|
+
throw new Error(`CanopyCmsService: invalid branchNamespace ${JSON.stringify(template)} -- ${why}.`);
|
|
323
|
+
};
|
|
324
|
+
const unknown = template.replaceAll(DEPLOYMENT_PLACEHOLDER, '').match(/\{[^}]*\}?/);
|
|
325
|
+
if (unknown) {
|
|
326
|
+
reject(`${JSON.stringify(unknown[0])} is not a placeholder; the only one is ${DEPLOYMENT_PLACEHOLDER}`);
|
|
327
|
+
}
|
|
328
|
+
const namespace = template.replaceAll(DEPLOYMENT_PLACEHOLDER, deploymentName);
|
|
329
|
+
const sample = `${namespace}x`;
|
|
330
|
+
if (!namespace.endsWith('/'))
|
|
331
|
+
reject('it must end with "/"');
|
|
332
|
+
if (!/^[\x21-\x7e]+$/.test(namespace))
|
|
333
|
+
reject('it is not printable ASCII without spaces');
|
|
334
|
+
if (/["%{}]/.test(namespace))
|
|
335
|
+
reject('it contains ", %, { or }');
|
|
336
|
+
if (/^(?:refs|heads|tags|remotes)\//i.test(namespace)) {
|
|
337
|
+
reject("it starts with 'refs/', 'heads/', 'tags/' or 'remotes/'");
|
|
338
|
+
}
|
|
339
|
+
if (sample.length > 250)
|
|
340
|
+
reject('a branch in it would be over 250 characters');
|
|
341
|
+
if (sample.includes('..'))
|
|
342
|
+
reject("it contains '..'");
|
|
343
|
+
if (sample.startsWith('/') || sample.includes('//'))
|
|
344
|
+
reject('it has an empty path component');
|
|
345
|
+
if (sample.startsWith('-'))
|
|
346
|
+
reject("a leading '-' parses as a git option");
|
|
347
|
+
if (/[~^:?*[\\]/.test(sample))
|
|
348
|
+
reject('it contains a character git forbids in a branch name');
|
|
349
|
+
if (sample.split('/').some((part) => part.startsWith('.') || part.endsWith('.lock'))) {
|
|
350
|
+
reject("a part of it starts with '.' or ends with '.lock'");
|
|
351
|
+
}
|
|
352
|
+
const sanitized = sample
|
|
353
|
+
.replace(/[^a-zA-Z0-9._-]/g, '-')
|
|
354
|
+
.replace(/-+/g, '-')
|
|
355
|
+
.replace(/^\.+/, '')
|
|
356
|
+
.replace(/(?<!\.)\.+$/, '');
|
|
357
|
+
const directoryPrefix = namespace
|
|
358
|
+
.replace(/[^a-zA-Z0-9._-]/g, '-')
|
|
359
|
+
.replace(/-+/g, '-')
|
|
360
|
+
.replace(/^\.+/, '')
|
|
361
|
+
.replace(/(?<!\.)\.+$/, '');
|
|
362
|
+
if ('canopycms-settings-'.startsWith(directoryPrefix) ||
|
|
363
|
+
sanitized.startsWith('canopycms-settings-')) {
|
|
364
|
+
reject('some branches in it would be canopycms-settings-* settings branches');
|
|
365
|
+
}
|
|
366
|
+
for (const [propName, branch] of [
|
|
367
|
+
['baseBranch', branches.baseBranch],
|
|
368
|
+
['settingsBranch', branches.settingsBranch],
|
|
369
|
+
]) {
|
|
370
|
+
const sanitize = (name) => name
|
|
371
|
+
.replace(/[^a-zA-Z0-9._-]/g, '-')
|
|
372
|
+
.replace(/-+/g, '-')
|
|
373
|
+
.replace(/^\.+/, '')
|
|
374
|
+
.replace(/(?<!\.)\.+$/, '');
|
|
375
|
+
if (branch !== undefined &&
|
|
376
|
+
(branch.startsWith(namespace) || sanitize(branch).startsWith(sanitize(namespace)))) {
|
|
377
|
+
reject(`the ${propName} ${JSON.stringify(branch)} is inside it, or shares a workspace directory ` +
|
|
378
|
+
'with a branch in it, and the namespace holds only the branches editors create');
|
|
379
|
+
}
|
|
380
|
+
if (branch !== undefined && namespace.startsWith(`${branch}/`)) {
|
|
381
|
+
reject(`the ${propName} ${JSON.stringify(branch)} is a branch named after part of it, so git can hold no branch in it`);
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
return namespace;
|
|
385
|
+
}
|
|
386
|
+
/**
|
|
387
|
+
* The most `legacyBranches` entries the gateway takes (`MAX_LEGACY_BRANCHES`,
|
|
388
|
+
* packages/canopycms/src/worker/branch-policy.ts). A local constant: this module imports nothing
|
|
389
|
+
* from core.
|
|
390
|
+
*/
|
|
391
|
+
const MAX_LEGACY_BRANCHES = 1024;
|
|
392
|
+
/**
|
|
393
|
+
* Refuse `legacyBranches` the GitHub gateway would refuse at boot in `namespace`, which would
|
|
394
|
+
* crash-loop it (`legacyBranchProblem` and the `BranchPolicy` constructor,
|
|
395
|
+
* packages/canopycms/src/worker/branch-policy.ts): more than {@link MAX_LEGACY_BRANCHES} entries,
|
|
396
|
+
* a repeated entry, and any entry that is not an exact branch name held to git's rules and the
|
|
397
|
+
* gateway's (`assertGatewayBranchName`), so none holds a `*`. No entry may be a settings branch,
|
|
398
|
+
* which the gateway pushes unlisted, start with the namespace, which it already writes, or be
|
|
399
|
+
* named after part of it, since git then holds no branch in it. A comma separates entries in the
|
|
400
|
+
* env file, so none may hold one. cms-deploy.test.ts holds this copy and core's to
|
|
401
|
+
* packages/canopycms/src/worker/legacy-branch-fixtures.ts; add a case there when you change
|
|
402
|
+
* either copy.
|
|
403
|
+
* @internal Exported for tests.
|
|
404
|
+
*/
|
|
405
|
+
export function assertValidLegacyBranches(entries, namespace) {
|
|
406
|
+
if (entries.length > MAX_LEGACY_BRANCHES) {
|
|
407
|
+
throw new Error(`CanopyCmsService: legacyBranches has ${entries.length} entries; the most it takes is ${MAX_LEGACY_BRANCHES}.`);
|
|
408
|
+
}
|
|
409
|
+
const seen = new Set();
|
|
410
|
+
return entries.map((entry) => {
|
|
411
|
+
const reject = (why) => {
|
|
412
|
+
throw new Error(`CanopyCmsService: invalid legacyBranches entry ${JSON.stringify(entry)} -- ${why}.`);
|
|
413
|
+
};
|
|
414
|
+
if (entry.includes(','))
|
|
415
|
+
reject('it contains a comma, which separates entries');
|
|
416
|
+
if (entry.includes('*'))
|
|
417
|
+
reject('an entry is an exact branch name, and * is not a wildcard here');
|
|
418
|
+
assertGatewayBranchName('legacyBranches entry', entry);
|
|
419
|
+
assertValidGitBranchName('legacyBranches entry', entry);
|
|
420
|
+
if (entry.startsWith('canopycms-settings-')) {
|
|
421
|
+
reject('it is a settings branch, which the gateway pushes without being listed');
|
|
422
|
+
}
|
|
423
|
+
if (entry.startsWith(namespace)) {
|
|
424
|
+
reject(`it starts with the branch namespace ${namespace}, so it is already in the namespace`);
|
|
425
|
+
}
|
|
426
|
+
if (namespace.startsWith(`${entry}/`)) {
|
|
427
|
+
reject(`it is a branch named after part of the branch namespace ${namespace}, so git can hold no branch in it`);
|
|
428
|
+
}
|
|
429
|
+
if (seen.has(entry))
|
|
430
|
+
reject('it is listed twice');
|
|
431
|
+
seen.add(entry);
|
|
432
|
+
return entry;
|
|
433
|
+
});
|
|
434
|
+
}
|
|
303
435
|
/**
|
|
304
436
|
* A Secrets Manager ARN carrying the ECS/CloudFormation JSON-field suffix,
|
|
305
437
|
* i.e. `arn:…:secret:name-AbCdEf:MY_KEY::` rather than `arn:…:secret:name-AbCdEf`.
|
|
@@ -682,6 +814,19 @@ export class CanopyCmsService extends Construct {
|
|
|
682
814
|
? assertValidGitBranchName('settingsBranch', props.settingsBranch)
|
|
683
815
|
: undefined;
|
|
684
816
|
const protectedBranches = assertValidProtectedBranches(props.protectedBranches ?? []);
|
|
817
|
+
// The construct stamps the namespace into both halves; an override in `environment` would reach
|
|
818
|
+
// the Lambda alone and be replaced there, so it is refused rather than silently ignored.
|
|
819
|
+
if (props.environment?.['CANOPYCMS_BRANCH_NAMESPACE'] !== undefined) {
|
|
820
|
+
throw new Error('CanopyCmsService: set the branch namespace with the branchNamespace prop, not ' +
|
|
821
|
+
'environment.CANOPYCMS_BRANCH_NAMESPACE: the prop reaches both the Lambda and the GitHub gateway.');
|
|
822
|
+
}
|
|
823
|
+
const branchNamespace = resolveBranchNamespaceProp(props.branchNamespace ?? DEFAULT_BRANCH_NAMESPACE_TEMPLATE, deploymentName, { baseBranch, settingsBranch });
|
|
824
|
+
// The gateway alone reads the list, from its own env file; `environment` reaches the Lambda.
|
|
825
|
+
if (props.environment?.['CANOPYCMS_LEGACY_BRANCHES'] !== undefined) {
|
|
826
|
+
throw new Error('CanopyCmsService: set legacy branches with the legacyBranches prop, not ' +
|
|
827
|
+
'environment.CANOPYCMS_LEGACY_BRANCHES: the prop reaches the GitHub gateway, and the environment only the Lambda.');
|
|
828
|
+
}
|
|
829
|
+
const legacyBranches = assertValidLegacyBranches(props.legacyBranches ?? [], branchNamespace);
|
|
685
830
|
const workerCapacity = props.workerCapacity ?? { type: 'on-demand' };
|
|
686
831
|
const workerInstanceTypes = workerCapacity.type === 'spot'
|
|
687
832
|
? (workerCapacity.instanceTypes ??
|
|
@@ -926,12 +1071,14 @@ export class CanopyCmsService extends Construct {
|
|
|
926
1071
|
// config. The fix lives in the image: Dockerfile.cms.template runs
|
|
927
1072
|
// `git config --system safe.directory '*'`.
|
|
928
1073
|
...props.environment,
|
|
929
|
-
// AFTER the spread, deliberately.
|
|
1074
|
+
// AFTER the spread, deliberately. The values are already the adopter's
|
|
930
1075
|
// own choice (an `environment` override is folded into `deploymentName`
|
|
931
|
-
// above and validated;
|
|
932
|
-
// is
|
|
933
|
-
//
|
|
1076
|
+
// above and validated; `branchNamespace` is resolved and validated
|
|
1077
|
+
// above; CANOPY_MODE is restricted to 'prod'), so nothing is taken
|
|
1078
|
+
// away - the placement is what stops the Lambda and the gateway's
|
|
1079
|
+
// env file holding different strings.
|
|
934
1080
|
CANOPYCMS_DEPLOYMENT_NAME: deploymentName,
|
|
1081
|
+
CANOPYCMS_BRANCH_NAMESPACE: branchNamespace,
|
|
935
1082
|
CANOPY_MODE: 'prod',
|
|
936
1083
|
},
|
|
937
1084
|
});
|
|
@@ -1180,8 +1327,11 @@ export class CanopyCmsService extends Construct {
|
|
|
1180
1327
|
// The SAME string the Lambda's environment gets above, including an
|
|
1181
1328
|
// `environment.CANOPYCMS_DEPLOYMENT_NAME` override - the two halves
|
|
1182
1329
|
// resolve one settings branch (`canopycms-settings-<name>`) between them,
|
|
1183
|
-
// and disagreeing here is what the runtime warning detects.
|
|
1184
|
-
|
|
1330
|
+
// and disagreeing here is what the runtime warning detects. The gateway resolves the same
|
|
1331
|
+
// settings branch from it, the only settings branch it pushes.
|
|
1332
|
+
['CANOPYCMS_DEPLOYMENT_NAME', deploymentName, 'both'],
|
|
1333
|
+
// The gateway enforces it and a split worker refuses it; the Lambda gets the same string.
|
|
1334
|
+
['CANOPYCMS_BRANCH_NAMESPACE', branchNamespace, 'gateway'],
|
|
1185
1335
|
// The AWS SDK JS v3 cannot resolve a region from IMDS on its own - without
|
|
1186
1336
|
// this a bare `SecretsManagerClient({})` crash-loops with "Region is
|
|
1187
1337
|
// missing".
|
|
@@ -1256,15 +1406,19 @@ export class CanopyCmsService extends Construct {
|
|
|
1256
1406
|
envEntries.push([WORKER_CAPACITY_ENV, 'spot', 'gateway']);
|
|
1257
1407
|
}
|
|
1258
1408
|
if (settingsBranch !== undefined) {
|
|
1259
|
-
// Only when explicitly set
|
|
1260
|
-
//
|
|
1261
|
-
//
|
|
1262
|
-
envEntries.push(['CANOPYCMS_SETTINGS_BRANCH', settingsBranch, '
|
|
1409
|
+
// Only when explicitly set: unset, the worker and the gateway both compute
|
|
1410
|
+
// `canopycms-settings-<name>`, and an empty value would name no branch. Both read it: the
|
|
1411
|
+
// gateway pushes no settings branch but the one the worker resolves.
|
|
1412
|
+
envEntries.push(['CANOPYCMS_SETTINGS_BRANCH', settingsBranch, 'both']);
|
|
1263
1413
|
}
|
|
1264
1414
|
if (protectedBranches.length > 0) {
|
|
1265
1415
|
// Only the gateway enforces it, and a split worker refuses it.
|
|
1266
1416
|
envEntries.push(['CANOPYCMS_PROTECTED_BRANCHES', protectedBranches.join(','), 'gateway']);
|
|
1267
1417
|
}
|
|
1418
|
+
if (legacyBranches.length > 0) {
|
|
1419
|
+
// Only the gateway enforces it, and a split worker refuses it.
|
|
1420
|
+
envEntries.push(['CANOPYCMS_LEGACY_BRANCHES', legacyBranches.join(','), 'gateway']);
|
|
1421
|
+
}
|
|
1268
1422
|
const envFile = (readers) => envEntries
|
|
1269
1423
|
.filter(([, , reader]) => readers.includes(reader))
|
|
1270
1424
|
.map(([name, value]) => `${name}=${assertEnvSafe(name, value)}`)
|
|
@@ -1287,7 +1441,7 @@ export class CanopyCmsService extends Construct {
|
|
|
1287
1441
|
...envEntries
|
|
1288
1442
|
.filter(([, value]) => Token.isUnresolved(value))
|
|
1289
1443
|
.map(([, value]) => ({ token: value, bytes: ENV_TOKEN_MAX_BYTES })),
|
|
1290
|
-
], protectedBranches.length, ...withoutShellComments('#!/bin/bash', 'set -euo pipefail', '', '# FAIL-FAST. Every step below is required for the worker to exist at', '# all, except the swap and the package upgrade, which are written as', '# conditions so that they cannot trip it. Until this trap existed, a', '# failure in any required step aborted user-data BEFORE the systemd unit was written -- leaving an instance', "# that runs, passes the ASG's EC2-only health check indefinitely, and", '# does nothing. cfn-signal is deliberately not used here (it would', '# prove nothing about READINESS, which is the argument recorded below),', '# but that argument never covered a boot-SCRIPT failure: `cdk deploy`', '# reported success while publishes queued on EFS, the auth cache went', '# stale and PRs stopped being created, until a human noticed the admin', '# panel showing the worker absent.', '#', '# Shutting down makes the instance fail its EC2 health check, so the ASG', '# replaces it -- which is the only automatic recovery available in this', '# topology. The echo lands in the console log, readable via', '# `aws ec2 get-console-output`, because the CloudWatch agent is itself', '# configured further down and cannot be relied on to exist yet.', 'trap \'echo "canopy-worker user-data FAILED at line $LINENO (exit $?)" >&2; shutdown -h now\' ERR', '', '# Bounded retry for the network-dependent steps. Package mirrors and S3', '# have transient failures; a single flake should cost seconds, not an', '# instance replacement. The first argument names the step in the message', '# an exhausted step leaves on the console.', 'retry() {', ' local step=$1 n=0', ' shift', ' until "$@"; do', ' n=$((n + 1))', ' if [ "$n" -ge 5 ]; then', ` echo "canopy-worker boot: '$step' failed after $n attempts" >&2`, ' return 1', ' fi', ' sleep $((n * 5))', ' done', '}', '', '# Swap, before the first dnf: loading the repository metadata takes more', '# memory than a t4g.nano has free after boot, and the kernel OOM-kills', '# dnf. It stays on afterwards at low swappiness, as headroom for git and', '# the CloudWatch agent rather than working memory. It is on the encrypted', '# root volume, so memory paged out of the worker is encrypted at rest.', '# fallocate is safe here: XFS, the root file system, supports', '# preallocated swap files. A failure only warns, because dnf may still fit', '# and a boot that stopped here would put the group into a replacement', '# loop. Each command carries `|| return 1` because errexit does not apply', '# inside a function called as a condition.', 'setup_swap() {', ' swapon --show=NAME --noheadings | grep -qx /swapfile && return 0', ' if [ "$(stat -c %s /swapfile 2>/dev/null)" != 1073741824 ]; then', ' # The file, plus 2 GiB for packages, logs and the GitHub mirror.', ' local avail_mib', ` avail_mib=$(df --output=avail -m / | tail -n 1 | tr -d ' ')`, ' case "$avail_mib" in ""|*[!0-9]*) return 1 ;; esac', ' if [ "$avail_mib" -lt 3072 ]; then', ' echo "canopy-worker boot: under 3 GiB free on /, so no swap file" >&2', ' return 1', ' fi', ' rm -f /swapfile || return 1', ' fallocate -l 1G /swapfile || return 1', ' fi', ' chmod 600 /swapfile || return 1', ' mkswap /swapfile || return 1', ' # swapon last, so that swap is never on when this returns 1.', " grep -qs '^/swapfile ' /etc/fstab || echo '/swapfile none swap defaults 0 0' >> /etc/fstab || return 1", ' mkdir -p /etc/sysctl.d || return 1', " echo 'vm.swappiness = 10' > /etc/sysctl.d/90-canopy-worker-swap.conf || return 1", ' sysctl -q -w vm.swappiness=10 || return 1', ' swapon /swapfile', '}', 'if ! setup_swap; then', ` echo "canopy-worker boot: 'swap' setup failed; continuing without swap" >&2`, 'fi', '', '# Patch first. AL2023 locks dnf to the repository release its AMI was', '# built from, so a relaunch from the AMI resolved at the last deploy', '# would otherwise install nothing newer than that AMI. Moving to the', '# latest release also makes it the one every install below resolves', '# from. The kernel is left out: a new one applies only at a reboot,', '# which nothing here performs, so it would cost boot time and change', '# nothing. Kernel fixes arrive with the AMI a deploy resolves.', '#', "# A failed upgrade does not stop the boot: the worker starts on the AMI's", '# packages and the next boot (by default the weekly recycle at the', '# latest) tries again. Failing here would put the group into a', '# replacement loop with no worker at all. The installs below then', '# resolve from the AMI release: 2023.7.20250428 carries all of them,', '# 2023.7.20250331 lacks nodejs22. The failure is written to worker.log', '# below, for the unpatched alarm.', 'WORKER_UNPATCHED=0', "if ! retry 'dnf upgrade' dnf upgrade --releasever=latest --exclude='kernel*' -y; then", ' WORKER_UNPATCHED=1', ` echo "canopy-worker boot: ${WORKER_UNPATCHED_LOG_PHRASE}" >&2`, 'fi', '', "retry 'dnf install git' dnf install -y git", "# Node comes from AL2023's own repos, NOT a piped third-party installer.", '# The previous boot curled the NodeSource RPM setup script straight into', '# bash, so every instance replacement -- which the ASG performs on every', '# `cdk deploy`, plus every spot interruption -- depended on a third party', '# being reachable. cms-deploy.test.ts asserts that URL never comes back,', '# which is why it is not spelled out here.', '#', '# nodejs22, not 20: Node 20 reached upstream EOL on 2026-04-30, and this', '# repo declares `engines.node: ">=22"` with .nvmrc `v22`, so the worker', '# was running an EOL runtime BELOW the floor its own code is tested at.', "# 22 (EOL 2027-04-30) matches .nvmrc, CI, and the transform Lambda's", '# NODEJS_22_X, so one runtime is tested everywhere.', '#', '# ExecStart uses the NAMESPACED /usr/bin/node-22 (see workerUnit in', '# worker-units.ts), not the bare `node`: AL2023 installs versioned binaries and', '# points `/usr/bin/node` at one of them through `alternatives`, whose', '# selection AWS documents as able to change at any time.', "retry 'dnf install nodejs22' dnf install -y nodejs22", '', '# Mount EFS through the same access point, at the same path, as the', '# Lambda (see EFS_MOUNT_PATH). amazon-efs-utils refuses an access-point', '# mount without tls (efs_utils_common/mount_options.py).', "retry 'dnf install amazon-efs-utils' dnf install -y amazon-efs-utils", `mkdir -p ${EFS_MOUNT_PATH}`, '# Retried: an `iam` mount also fetches instance-role credentials.', `retry 'EFS mount' mount -t efs -o ${efsMountOptions} ${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH}`, '# Persist the mount across instance reboots: user-data runs once per', '# instance, so without an fstab entry a plain reboot leaves /mnt/efs an', '# empty local dir and the worker would clone a divergent remote.git', '# onto the instance disk, invisible to the Lambda.', `echo '${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH} efs _netdev,${efsMountOptions} 0 0' >> /etc/fstab`, '', '# The bundle runs only if its sha256 is the one given here (synthesized,', '# or the workerCode parameter); a mismatch fails the boot under the trap', '# above.', `retry 'worker bundle download' aws s3 cp s3://${workerBundle.bucketName}/${workerBundle.objectKey} ${WORKER_BUNDLE_DOWNLOAD_PATH}`, `echo '${workerBundle.sha256} ${WORKER_BUNDLE_DOWNLOAD_PATH}' | sha256sum -c -`, '# Root-owned, and read-only to the services (ProtectSystem=strict in worker-units.ts).', `install -D -m 0644 ${WORKER_BUNDLE_DOWNLOAD_PATH} /opt/canopy-worker/index.js`, '# The worker bundle is ESM (esbuild --format=esm). Without this marker a', '# .js file is CommonJS by default and the import statement fails.', '# Node >=22.7 auto-detects module syntax and would mask its absence, and', '# this instance now runs node-22 -- so the marker is kept BECAUSE that', '# auto-detection is a fallback we should not depend on, not because the', '# runtime here still needs it.', `echo '{"type":"module"}' > /opt/canopy-worker/package.json`, '', '# The units and scripts below install from, checked as the bundle is and', '# unpacked where only root can read or write.', `retry 'worker instance files download' aws s3 cp s3://${workerBundle.instanceFiles.bucketName}/${workerBundle.instanceFiles.objectKey} ${INSTANCE_FILES_DOWNLOAD_PATH}`, `echo '${workerBundle.instanceFiles.sha256} ${INSTANCE_FILES_DOWNLOAD_PATH}' | sha256sum -c -`, `install -d -o root -g root -m 0700 ${INSTANCE_FILES_DIR}`, `(cd ${INSTANCE_FILES_DIR} && bash ${INSTANCE_FILES_DOWNLOAD_PATH})`, '', "# The worker's environment: nothing secret.", `cat > ${WORKER_INSTALL_DIR}/.env << '${ENV_HEREDOC_DELIMITER}'`, workerEnv, ENV_HEREDOC_DELIMITER, "# The secrets' names, root's alone until the split gives the gateway's", '# group read below: created with its mode before anything is written to', '# it, so it is never world-readable. The single unit reads both files.', `install -m 0600 -o root -g root /dev/null ${WORKER_INSTALL_DIR}/gateway.env`, `cat > ${WORKER_INSTALL_DIR}/gateway.env << '${ENV_HEREDOC_DELIMITER}'`, gatewayEnv, ENV_HEREDOC_DELIMITER, '', '# The units, chosen by the contract the bundle states in its first', `# lines (\`${WORKER_CONTRACT_BANNER_PREFIX}<n>\`), read after the sha256 check above,`, '# so it is as trusted as the bundle. A bundle at the split contract or', '# later gets the split: a worker with no network and no secret, and the', '# GitHub gateway holding them. An older one gets the single unit it was', '# built for: the split would cut it off from GitHub. Every bundle roll', '# replaces the instance, so each boot matches its own bundle.', 'WORKER_BUNDLE_CONTRACT=$(head -n 5 /opt/canopy-worker/index.js |', ` sed -n 's|^${WORKER_CONTRACT_BANNER_PREFIX}\\([0-9]\\{1,4\\}\\)$|\\1|p' | head -n 1 || true)`, `if [ "\${WORKER_BUNDLE_CONTRACT:-0}" -ge ${GATEWAY_SPLIT_CONTRACT} ]; then`, ' WORKER_SPLIT=1', '', ' # No core dumps on this instance. A piped core_pattern hands a crashing', " # process's memory and environment to a root helper, which journals", " # the environment where the worker's groups can read it; the gateway", ' # refuses to start unless this holds. systemd-sysctl has already run', ' # by now, so the file is applied here as well as at every boot.', ' # fb_tunnels_only_for_init_net keeps fallback tunnel devices out of', " # the worker's network namespace, which must hold only lo. -e skips a", ' # key this kernel lacks: the gateway refuses to start under a piped', ' # core_pattern, and the worker with any interface but lo.', ' mkdir -p /etc/sysctl.d', " cat > /etc/sysctl.d/99-canopy.conf << 'SYSCTLEOF'", 'kernel.core_pattern = core', 'fs.suid_dumpable = 0', 'net.core.fb_tunnels_only_for_init_net = 1', 'SYSCTLEOF', ' sysctl -q -e -p /etc/sysctl.d/99-canopy.conf', ' # PID 1 otherwise gives every service an unlimited core limit', ' # (DefaultLimitCORE=, systemd-system.conf(5)), so', ' # core_pattern=core would leave core files from any crashing daemon.', ' mkdir -p /etc/systemd/system.conf.d', " printf '[Manager]\\nDefaultLimitCORE=0\\n' > /etc/systemd/system.conf.d/canopy.conf", '', " # The gateway's user: a system account, so its uid is stable and", " # gateway.env can be its group's before the service first runs.", ` getent passwd ${GATEWAY_USER} >/dev/null ||`, ` useradd --system --no-create-home --shell /sbin/nologin ${GATEWAY_USER}`, ` chown root:${GATEWAY_USER} ${WORKER_INSTALL_DIR}/gateway.env`, ` chmod 0640 ${WORKER_INSTALL_DIR}/gateway.env`, '', ...['canopy-worker.service', 'canopy-github.service', 'canopy-github.socket'].map((unit) => ` install -m 0644 ${INSTANCE_FILES_DIR}/${unit} /etc/systemd/system/${unit}`), '', ' # The gateway log, root-owned (LOG_RULE, worker-units.ts), and readable', ' # by root alone.', ` install -d -o root -g root -m 0755 ${GATEWAY_LOG_DIR}`, ` install -o root -g root -m 0640 /dev/null ${GATEWAY_LOG}`, '', ' # canopy-inspect, for an operator on EFS: see canopyInspectScript', ' # (worker-units.ts).', ` install -m 0755 ${INSTANCE_FILES_DIR}/canopy-inspect ${CANOPY_INSPECT}`, ` install -m 0755 ${INSTANCE_FILES_DIR}/inspect-checks ${INSPECT_CHECKS}`, ` install -m 0644 ${INSTANCE_FILES_DIR}/inspect.rc ${INSPECT_RC}`, 'else', ' WORKER_SPLIT=0', ` install -m 0644 ${INSTANCE_FILES_DIR}/canopy-worker-single-process.service /etc/systemd/system/canopy-worker.service`, 'fi', '', '# The worker log, root-owned and created before the first start (LOG_RULE,', '# worker-units.ts). 0644, so it stays', '# readable without sudo.', `install -d -o root -g root -m 0755 ${WORKER_LOG_DIR}`, `install -o root -g root -m 0644 /dev/null ${WORKER_LOG}`, '', '# In the worker log format (canopycms worker/log.ts), so the CloudWatch', '# agent ships it as its own event: the agent reads a file it has no saved', '# position for from the start (its `from_beginning` defaults to true).', 'if [ "$WORKER_UNPATCHED" = 1 ]; then', ` echo "$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) ERROR canopy-worker boot: 'dnf upgrade' failed; ${WORKER_UNPATCHED_LOG_PHRASE}" >> ${WORKER_LOG}`, 'fi', '', '# Start the units; daemon-reload also reads the core-limit drop-in. The', '# gateway first, so its secret reads and first GitHub fetch start before', "# the worker's first call.", 'systemctl daemon-reload', 'if [ "$WORKER_SPLIT" = 1 ]; then', ' systemctl enable --now canopy-github.socket canopy-github.service', 'fi', 'systemctl enable canopy-worker', 'systemctl start canopy-worker', '', '# ---- CloudWatch log shipping ----', '# Placed AFTER worker start: with set -euo pipefail, a package/agent', '# failure here must not prevent the worker from running (shipping is', '# best-effort).', '#', '# DISARM THE FAIL-FAST TRAP AND errexit FIRST. Everything above this line', '# is required for the worker to exist at all, so a failure there should', '# replace the instance. Nothing below it is: the worker is already', '# running and healthy by this point. Leaving the trap armed would let a', '# package-mirror outage during the agent install shut down a perfectly', '# good worker -- and the ASG would relaunch straight into the same', '# outage, turning degraded log shipping into a replacement loop with the', '# worker down for its duration. That is strictly worse than the', '# best-effort behaviour this section has always documented.', 'trap - ERR', 'set +e', "retry 'dnf install amazon-cloudwatch-agent logrotate' dnf install -y amazon-cloudwatch-agent logrotate", '', '# Bound on-disk growth; copytruncate keeps the fd the CW agent tails valid', '# (tiny copy->truncate loss window is acceptable for diagnostic logs).', `cat > /etc/logrotate.d/canopy-worker << 'ROTEOF'`, `${WORKER_LOG} ${GATEWAY_LOG} {`, ' size 10M', ' rotate 5', ' compress', ' copytruncate', ' missingok', ' notifempty', '}', 'ROTEOF', '# The config above only fires when logrotate actually runs; AL2023', '# presets may leave logrotate.timer disabled, and without it the size', '# cap never triggers and worker.log grows until the nano disk fills.', '# --now is idempotent if the timer is already enabled/running.', 'systemctl enable --now logrotate.timer', '', '# No retention_in_days here: CDK owns retention on the pre-created group.', `cat > /opt/aws/amazon-cloudwatch-agent/etc/canopy-worker-logs.json << 'CWEOF'`, '{', ' "logs": {', ' "logs_collected": {', ' "files": {', ' "collect_list": [', ...[
|
|
1444
|
+
], { protectedBranches: protectedBranches.length, legacyBranches: legacyBranches.length }, ...withoutShellComments('#!/bin/bash', 'set -euo pipefail', '', '# FAIL-FAST. Every step below is required for the worker to exist at', '# all, except the swap and the package upgrade, which are written as', '# conditions so that they cannot trip it. Until this trap existed, a', '# failure in any required step aborted user-data BEFORE the systemd unit was written -- leaving an instance', "# that runs, passes the ASG's EC2-only health check indefinitely, and", '# does nothing. cfn-signal is deliberately not used here (it would', '# prove nothing about READINESS, which is the argument recorded below),', '# but that argument never covered a boot-SCRIPT failure: `cdk deploy`', '# reported success while publishes queued on EFS, the auth cache went', '# stale and PRs stopped being created, until a human noticed the admin', '# panel showing the worker absent.', '#', '# Shutting down makes the instance fail its EC2 health check, so the ASG', '# replaces it -- which is the only automatic recovery available in this', '# topology. The echo lands in the console log, readable via', '# `aws ec2 get-console-output`, because the CloudWatch agent is itself', '# configured further down and cannot be relied on to exist yet.', 'trap \'echo "canopy-worker user-data FAILED at line $LINENO (exit $?)" >&2; shutdown -h now\' ERR', '', '# Bounded retry for the network-dependent steps. Package mirrors and S3', '# have transient failures; a single flake should cost seconds, not an', '# instance replacement. The first argument names the step in the message', '# an exhausted step leaves on the console.', 'retry() {', ' local step=$1 n=0', ' shift', ' until "$@"; do', ' n=$((n + 1))', ' if [ "$n" -ge 5 ]; then', ` echo "canopy-worker boot: '$step' failed after $n attempts" >&2`, ' return 1', ' fi', ' sleep $((n * 5))', ' done', '}', '', '# Swap, before the first dnf: loading the repository metadata takes more', '# memory than a t4g.nano has free after boot, and the kernel OOM-kills', '# dnf. It stays on afterwards at low swappiness, as headroom for git and', '# the CloudWatch agent rather than working memory. It is on the encrypted', '# root volume, so memory paged out of the worker is encrypted at rest.', '# fallocate is safe here: XFS, the root file system, supports', '# preallocated swap files. A failure only warns, because dnf may still fit', '# and a boot that stopped here would put the group into a replacement', '# loop. Each command carries `|| return 1` because errexit does not apply', '# inside a function called as a condition.', 'setup_swap() {', ' swapon --show=NAME --noheadings | grep -qx /swapfile && return 0', ' if [ "$(stat -c %s /swapfile 2>/dev/null)" != 1073741824 ]; then', ' # The file, plus 2 GiB for packages, logs and the GitHub mirror.', ' local avail_mib', ` avail_mib=$(df --output=avail -m / | tail -n 1 | tr -d ' ')`, ' case "$avail_mib" in ""|*[!0-9]*) return 1 ;; esac', ' if [ "$avail_mib" -lt 3072 ]; then', ' echo "canopy-worker boot: under 3 GiB free on /, so no swap file" >&2', ' return 1', ' fi', ' rm -f /swapfile || return 1', ' fallocate -l 1G /swapfile || return 1', ' fi', ' chmod 600 /swapfile || return 1', ' mkswap /swapfile || return 1', ' # swapon last, so that swap is never on when this returns 1.', " grep -qs '^/swapfile ' /etc/fstab || echo '/swapfile none swap defaults 0 0' >> /etc/fstab || return 1", ' mkdir -p /etc/sysctl.d || return 1', " echo 'vm.swappiness = 10' > /etc/sysctl.d/90-canopy-worker-swap.conf || return 1", ' sysctl -q -w vm.swappiness=10 || return 1', ' swapon /swapfile', '}', 'if ! setup_swap; then', ` echo "canopy-worker boot: 'swap' setup failed; continuing without swap" >&2`, 'fi', '', '# Patch first. AL2023 locks dnf to the repository release its AMI was', '# built from, so a relaunch from the AMI resolved at the last deploy', '# would otherwise install nothing newer than that AMI. Moving to the', '# latest release also makes it the one every install below resolves', '# from. The kernel is left out: a new one applies only at a reboot,', '# which nothing here performs, so it would cost boot time and change', '# nothing. Kernel fixes arrive with the AMI a deploy resolves.', '#', "# A failed upgrade does not stop the boot: the worker starts on the AMI's", '# packages and the next boot (by default the weekly recycle at the', '# latest) tries again. Failing here would put the group into a', '# replacement loop with no worker at all. The installs below then', '# resolve from the AMI release: 2023.7.20250428 carries all of them,', '# 2023.7.20250331 lacks nodejs22. The failure is written to worker.log', '# below, for the unpatched alarm.', 'WORKER_UNPATCHED=0', "if ! retry 'dnf upgrade' dnf upgrade --releasever=latest --exclude='kernel*' -y; then", ' WORKER_UNPATCHED=1', ` echo "canopy-worker boot: ${WORKER_UNPATCHED_LOG_PHRASE}" >&2`, 'fi', '', "retry 'dnf install git' dnf install -y git", "# Node comes from AL2023's own repos, NOT a piped third-party installer.", '# The previous boot curled the NodeSource RPM setup script straight into', '# bash, so every instance replacement -- which the ASG performs on every', '# `cdk deploy`, plus every spot interruption -- depended on a third party', '# being reachable. cms-deploy.test.ts asserts that URL never comes back,', '# which is why it is not spelled out here.', '#', '# nodejs22, not 20: Node 20 reached upstream EOL on 2026-04-30, and this', '# repo declares `engines.node: ">=22"` with .nvmrc `v22`, so the worker', '# was running an EOL runtime BELOW the floor its own code is tested at.', "# 22 (EOL 2027-04-30) matches .nvmrc, CI, and the transform Lambda's", '# NODEJS_22_X, so one runtime is tested everywhere.', '#', '# ExecStart uses the NAMESPACED /usr/bin/node-22 (see workerUnit in', '# worker-units.ts), not the bare `node`: AL2023 installs versioned binaries and', '# points `/usr/bin/node` at one of them through `alternatives`, whose', '# selection AWS documents as able to change at any time.', "retry 'dnf install nodejs22' dnf install -y nodejs22", '', '# Mount EFS through the same access point, at the same path, as the', '# Lambda (see EFS_MOUNT_PATH). amazon-efs-utils refuses an access-point', '# mount without tls (efs_utils_common/mount_options.py).', "retry 'dnf install amazon-efs-utils' dnf install -y amazon-efs-utils", `mkdir -p ${EFS_MOUNT_PATH}`, '# Retried: an `iam` mount also fetches instance-role credentials.', `retry 'EFS mount' mount -t efs -o ${efsMountOptions} ${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH}`, '# Persist the mount across instance reboots: user-data runs once per', '# instance, so without an fstab entry a plain reboot leaves /mnt/efs an', '# empty local dir and the worker would clone a divergent remote.git', '# onto the instance disk, invisible to the Lambda.', `echo '${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH} efs _netdev,${efsMountOptions} 0 0' >> /etc/fstab`, '', '# The bundle runs only if its sha256 is the one given here (synthesized,', '# or the workerCode parameter); a mismatch fails the boot under the trap', '# above.', `retry 'worker bundle download' aws s3 cp s3://${workerBundle.bucketName}/${workerBundle.objectKey} ${WORKER_BUNDLE_DOWNLOAD_PATH}`, `echo '${workerBundle.sha256} ${WORKER_BUNDLE_DOWNLOAD_PATH}' | sha256sum -c -`, '# Root-owned, and read-only to the services (ProtectSystem=strict in worker-units.ts).', `install -D -m 0644 ${WORKER_BUNDLE_DOWNLOAD_PATH} /opt/canopy-worker/index.js`, '# The worker bundle is ESM (esbuild --format=esm). Without this marker a', '# .js file is CommonJS by default and the import statement fails.', '# Node >=22.7 auto-detects module syntax and would mask its absence, and', '# this instance now runs node-22 -- so the marker is kept BECAUSE that', '# auto-detection is a fallback we should not depend on, not because the', '# runtime here still needs it.', `echo '{"type":"module"}' > /opt/canopy-worker/package.json`, '', '# The units and scripts below install from, checked as the bundle is and', '# unpacked where only root can read or write.', `retry 'worker instance files download' aws s3 cp s3://${workerBundle.instanceFiles.bucketName}/${workerBundle.instanceFiles.objectKey} ${INSTANCE_FILES_DOWNLOAD_PATH}`, `echo '${workerBundle.instanceFiles.sha256} ${INSTANCE_FILES_DOWNLOAD_PATH}' | sha256sum -c -`, `install -d -o root -g root -m 0700 ${INSTANCE_FILES_DIR}`, `(cd ${INSTANCE_FILES_DIR} && bash ${INSTANCE_FILES_DOWNLOAD_PATH})`, '', "# The worker's environment: nothing secret.", `cat > ${WORKER_INSTALL_DIR}/.env << '${ENV_HEREDOC_DELIMITER}'`, workerEnv, ENV_HEREDOC_DELIMITER, "# The secrets' names, root's alone until the split gives the gateway's", '# group read below: created with its mode before anything is written to', '# it, so it is never world-readable. The single unit reads both files.', `install -m 0600 -o root -g root /dev/null ${WORKER_INSTALL_DIR}/gateway.env`, `cat > ${WORKER_INSTALL_DIR}/gateway.env << '${ENV_HEREDOC_DELIMITER}'`, gatewayEnv, ENV_HEREDOC_DELIMITER, '', '# The units, chosen by the contract the bundle states in its first', `# lines (\`${WORKER_CONTRACT_BANNER_PREFIX}<n>\`), read after the sha256 check above,`, '# so it is as trusted as the bundle. A bundle at the split contract or', '# later gets the split: a worker with no network and no secret, and the', '# GitHub gateway holding them. An older one gets the single unit it was', '# built for: the split would cut it off from GitHub. Every bundle roll', '# replaces the instance, so each boot matches its own bundle.', 'WORKER_BUNDLE_CONTRACT=$(head -n 5 /opt/canopy-worker/index.js |', ` sed -n 's|^${WORKER_CONTRACT_BANNER_PREFIX}\\([0-9]\\{1,4\\}\\)$|\\1|p' | head -n 1 || true)`, `if [ "\${WORKER_BUNDLE_CONTRACT:-0}" -ge ${GATEWAY_SPLIT_CONTRACT} ]; then`, ' WORKER_SPLIT=1', '', ' # No core dumps on this instance. A piped core_pattern hands a crashing', " # process's memory and environment to a root helper, which journals", " # the environment where the worker's groups can read it; the gateway", ' # refuses to start unless this holds. systemd-sysctl has already run', ' # by now, so the file is applied here as well as at every boot.', ' # fb_tunnels_only_for_init_net keeps fallback tunnel devices out of', " # the worker's network namespace, which must hold only lo. -e skips a", ' # key this kernel lacks: the gateway refuses to start under a piped', ' # core_pattern, and the worker with any interface but lo.', ' mkdir -p /etc/sysctl.d', " cat > /etc/sysctl.d/99-canopy.conf << 'SYSCTLEOF'", 'kernel.core_pattern = core', 'fs.suid_dumpable = 0', 'net.core.fb_tunnels_only_for_init_net = 1', 'SYSCTLEOF', ' sysctl -q -e -p /etc/sysctl.d/99-canopy.conf', ' # PID 1 otherwise gives every service an unlimited core limit', ' # (DefaultLimitCORE=, systemd-system.conf(5)), so', ' # core_pattern=core would leave core files from any crashing daemon.', ' mkdir -p /etc/systemd/system.conf.d', " printf '[Manager]\\nDefaultLimitCORE=0\\n' > /etc/systemd/system.conf.d/canopy.conf", '', " # The gateway's user: a system account, so its uid is stable and", " # gateway.env can be its group's before the service first runs.", ` getent passwd ${GATEWAY_USER} >/dev/null ||`, ` useradd --system --no-create-home --shell /sbin/nologin ${GATEWAY_USER}`, ` chown root:${GATEWAY_USER} ${WORKER_INSTALL_DIR}/gateway.env`, ` chmod 0640 ${WORKER_INSTALL_DIR}/gateway.env`, '', ...['canopy-worker.service', 'canopy-github.service', 'canopy-github.socket'].map((unit) => ` install -m 0644 ${INSTANCE_FILES_DIR}/${unit} /etc/systemd/system/${unit}`), '', ' # The gateway log, root-owned (LOG_RULE, worker-units.ts), and readable', ' # by root alone.', ` install -d -o root -g root -m 0755 ${GATEWAY_LOG_DIR}`, ` install -o root -g root -m 0640 /dev/null ${GATEWAY_LOG}`, '', ' # canopy-inspect, for an operator on EFS: see canopyInspectScript', ' # (worker-units.ts).', ` install -m 0755 ${INSTANCE_FILES_DIR}/canopy-inspect ${CANOPY_INSPECT}`, ` install -m 0755 ${INSTANCE_FILES_DIR}/inspect-checks ${INSPECT_CHECKS}`, ` install -m 0644 ${INSTANCE_FILES_DIR}/inspect.rc ${INSPECT_RC}`, 'else', ' WORKER_SPLIT=0', ` install -m 0644 ${INSTANCE_FILES_DIR}/canopy-worker-single-process.service /etc/systemd/system/canopy-worker.service`, 'fi', '', '# The worker log, root-owned and created before the first start (LOG_RULE,', '# worker-units.ts). 0644, so it stays', '# readable without sudo.', `install -d -o root -g root -m 0755 ${WORKER_LOG_DIR}`, `install -o root -g root -m 0644 /dev/null ${WORKER_LOG}`, '', '# In the worker log format (canopycms worker/log.ts), so the CloudWatch', '# agent ships it as its own event: the agent reads a file it has no saved', '# position for from the start (its `from_beginning` defaults to true).', 'if [ "$WORKER_UNPATCHED" = 1 ]; then', ` echo "$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) ERROR canopy-worker boot: 'dnf upgrade' failed; ${WORKER_UNPATCHED_LOG_PHRASE}" >> ${WORKER_LOG}`, 'fi', '', '# Start the units; daemon-reload also reads the core-limit drop-in. The', '# gateway first, so its secret reads and first GitHub fetch start before', "# the worker's first call.", 'systemctl daemon-reload', 'if [ "$WORKER_SPLIT" = 1 ]; then', ' systemctl enable --now canopy-github.socket canopy-github.service', 'fi', 'systemctl enable canopy-worker', 'systemctl start canopy-worker', '', '# ---- CloudWatch log shipping ----', '# Placed AFTER worker start: with set -euo pipefail, a package/agent', '# failure here must not prevent the worker from running (shipping is', '# best-effort).', '#', '# DISARM THE FAIL-FAST TRAP AND errexit FIRST. Everything above this line', '# is required for the worker to exist at all, so a failure there should', '# replace the instance. Nothing below it is: the worker is already', '# running and healthy by this point. Leaving the trap armed would let a', '# package-mirror outage during the agent install shut down a perfectly', '# good worker -- and the ASG would relaunch straight into the same', '# outage, turning degraded log shipping into a replacement loop with the', '# worker down for its duration. That is strictly worse than the', '# best-effort behaviour this section has always documented.', 'trap - ERR', 'set +e', "retry 'dnf install amazon-cloudwatch-agent logrotate' dnf install -y amazon-cloudwatch-agent logrotate", '', '# Bound on-disk growth; copytruncate keeps the fd the CW agent tails valid', '# (tiny copy->truncate loss window is acceptable for diagnostic logs).', `cat > /etc/logrotate.d/canopy-worker << 'ROTEOF'`, `${WORKER_LOG} ${GATEWAY_LOG} {`, ' size 10M', ' rotate 5', ' compress', ' copytruncate', ' missingok', ' notifempty', '}', 'ROTEOF', '# The config above only fires when logrotate actually runs; AL2023', '# presets may leave logrotate.timer disabled, and without it the size', '# cap never triggers and worker.log grows until the nano disk fills.', '# --now is idempotent if the timer is already enabled/running.', 'systemctl enable --now logrotate.timer', '', '# No retention_in_days here: CDK owns retention on the pre-created group.', `cat > /opt/aws/amazon-cloudwatch-agent/etc/canopy-worker-logs.json << 'CWEOF'`, '{', ' "logs": {', ' "logs_collected": {', ' "files": {', ' "collect_list": [', ...[
|
|
1291
1445
|
{ file: WORKER_LOG, stream: '{instance_id}' },
|
|
1292
1446
|
// The gateway's own record of what reached GitHub, which the worker
|
|
1293
1447
|
// cannot write.
|
|
@@ -48,7 +48,7 @@ export declare const WORKER_CONTRACT_BANNER_PREFIX = "// canopy-worker-contract:
|
|
|
48
48
|
* bundle as `worker/dist/index.js.contract`, so a change-set gate can compare
|
|
49
49
|
* the two before rolling a bundle. See {@link WORKER_CONTRACT_REQUIREMENTS}.
|
|
50
50
|
*/
|
|
51
|
-
export declare const WORKER_CONTRACT_VERSION:
|
|
51
|
+
export declare const WORKER_CONTRACT_VERSION: 3;
|
|
52
52
|
/** The unit's `Environment=` variable carrying its worker contract version. */
|
|
53
53
|
export declare const WORKER_CONTRACT_ENV = "CANOPYCMS_WORKER_CONTRACT";
|
|
54
54
|
/**
|
|
@@ -63,6 +63,7 @@ export const WORKER_CONTRACT_BANNER_PREFIX = '// canopy-worker-contract: ';
|
|
|
63
63
|
const WORKER_CONTRACT_REQUIREMENTS = [
|
|
64
64
|
'StateDirectory=canopy-worker',
|
|
65
65
|
`canopy-github.socket and canopy-github.service, and ${GITHUB_GATEWAY_SOCKET_ENV} on the worker unit`,
|
|
66
|
+
'CANOPYCMS_DEPLOYMENT_NAME, and CANOPYCMS_SETTINGS_BRANCH when set, in gateway.env',
|
|
66
67
|
];
|
|
67
68
|
/**
|
|
68
69
|
* The worker contract version: stamped into the unit by the construct (and,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "canopycms-cdk",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.8.1",
|
|
4
4
|
"description": "AWS CDK constructs and EC2 worker for CanopyCMS deployment",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -34,7 +34,7 @@
|
|
|
34
34
|
"peerDependencies": {
|
|
35
35
|
"aws-cdk-lib": "^2.260.0",
|
|
36
36
|
"constructs": "^10.0.0",
|
|
37
|
-
"canopycms": "0.
|
|
37
|
+
"canopycms": "0.8.1"
|
|
38
38
|
},
|
|
39
39
|
"peerDependenciesMeta": {
|
|
40
40
|
"aws-cdk-lib": {
|
|
@@ -60,8 +60,8 @@
|
|
|
60
60
|
"tsx": "^4.19.0",
|
|
61
61
|
"typescript": "^5.6.3",
|
|
62
62
|
"vitest": "^4.1.0",
|
|
63
|
-
"canopycms
|
|
64
|
-
"canopycms": "0.
|
|
63
|
+
"canopycms": "0.8.1",
|
|
64
|
+
"canopycms-auth-clerk": "0.8.1"
|
|
65
65
|
},
|
|
66
66
|
"scripts": {
|
|
67
67
|
"build": "rm -rf dist && tsc && node ../../scripts/add-js-extensions.mjs dist",
|
|
@@ -45,7 +45,7 @@ StateDirectoryMode=0700
|
|
|
45
45
|
# 0640 root:canopy-github: the secrets, unreadable to the worker.
|
|
46
46
|
EnvironmentFile=/opt/canopy-worker/gateway.env
|
|
47
47
|
# What this unit provides; a bundle needing more refuses to start.
|
|
48
|
-
Environment=CANOPYCMS_WORKER_CONTRACT=
|
|
48
|
+
Environment=CANOPYCMS_WORKER_CONTRACT=3
|
|
49
49
|
# Sandbox. Writable: the state dir and a private /tmp. No view of EFS,
|
|
50
50
|
# which the gateway checks at boot, as it checks that cores are not
|
|
51
51
|
# piped to a helper and that its core limit is zero.
|
|
@@ -45,7 +45,7 @@ StandardError=append:/var/log/canopy-worker/worker.log
|
|
|
45
45
|
StateDirectory=canopy-worker
|
|
46
46
|
EnvironmentFile=/opt/canopy-worker/.env
|
|
47
47
|
# What this unit provides; a bundle needing more refuses to start.
|
|
48
|
-
Environment=CANOPYCMS_WORKER_CONTRACT=
|
|
48
|
+
Environment=CANOPYCMS_WORKER_CONTRACT=3
|
|
49
49
|
# The GitHub gateway, its only way to GitHub, the Clerk API and the
|
|
50
50
|
# instance role.
|
|
51
51
|
Environment=CANOPYCMS_GITHUB_GATEWAY_SOCKET=/run/canopy-github/gateway.sock
|