canopycms-cdk 0.0.68-int.108 → 0.0.68-int.110

package/README.md

@@ -9,10 +9,11 @@ AWS CDK constructs and EC2 worker for deploying CanopyCMS.
 **`CanopyCmsService`** — Core infrastructure (required):
 
 - VPC (2 AZs, public + private subnets, no NAT Gateway)
-- EFS filesystem with `/workspace` access point
+- EFS filesystem with `/workspace` access point, TLS and IAM required, backed up daily (`efsBackup`)
 - Lambda function (Docker image, EFS mount, private subnet, no internet)
 - Lambda Function URL (for CloudFront origin)
-- EC2 Worker (on-demand t4g.nano in ASG, spot opt-in, public subnet, EFS mount, drained before replacement)
+- EC2 Worker (on-demand t4g.nano in ASG, spot opt-in, public subnet, EFS mount, drained before replacement, replaced weekly, [hardened](../../docs/deploying-to-aws.md#the-worker-instance))
+- `workerCode: { source: 'parameter' }`: lets CI roll the worker with a parameter-only change set; see [Rolling the worker from CI](../../docs/deploying-to-aws.md#rolling-the-worker-from-ci)
 - Security groups and IAM roles (least-privilege)
 - `attachTo(distribution)`: adds the editor's `/edit`, `/edit/*` and `/api/canopycms/*` behaviors (and, optionally, the CMS build's `assetPrefix` and the preview route) to a CloudFront distribution you already own
 
@@ -79,6 +80,7 @@ See [ARCHITECTURE.md](../../ARCHITECTURE.md#deployment-architecture) for details
 | EC2 t4g.nano on-demand (ASG 1/1/1) | ~$3              |
 | Lambda (editors only)              | ~$1-5            |
 | EFS (small repo)                   | ~$1              |
+| EFS backup (~$0.05/GB-month)       | cents            |
 | CloudFront (low traffic)           | ~$1              |
 | **Total**                          | **~$6-10/month** |
 

package/dist/constructs/cms-service.d.ts

@@ -1,6 +1,8 @@
 import { Construct } from 'constructs';
-import { Duration, RemovalPolicy, aws_ec2 as ec2, aws_efs as efs, aws_cloudfront as cloudfront, aws_iam as iam, aws_lambda as lambda, aws_autoscaling as autoscaling, aws_logs as logs } from 'aws-cdk-lib';
+import { CfnParameter, Duration, RemovalPolicy, aws_cloudwatch as cloudwatch, aws_ec2 as ec2, aws_efs as efs, aws_cloudfront as cloudfront, aws_iam as iam, aws_lambda as lambda, aws_autoscaling as autoscaling, aws_s3 as s3, aws_logs as logs } from 'aws-cdk-lib';
 import type { IBucket } from 'aws-cdk-lib/aws-s3';
+import type * as sns from 'aws-cdk-lib/aws-sns';
+import type { WorkerCode } from './worker-bundle.js';
 import type { CanopyCmsAttachOptions } from './editor-routing.js';
 /**
  * Default CMS Lambda timeout.
@@ -93,6 +95,23 @@ export interface CanopyCmsServiceProps {
      * this bounds only a worker that hangs.
      */
     workerTerminationHeartbeat?: Duration;
+    /**
+     * How long a worker instance may run before Auto Scaling replaces it
+     * (default 7 days; between 1 and 365 days; `null` never replaces it on
+     * age). Each replacement boots on the latest Amazon Linux packages, the
+     * kernel excepted, which is how a worker no deploy has touched still gets
+     * patched.
+     *
+     * A replacement pauses only the worker's half: saves keep working, while
+     * publishing, pull requests and sync wait for the new instance to boot.
+     */
+    workerMaxInstanceLifetime?: Duration | null;
+    /**
+     * Where the worker's bundle comes from (default `{ source: 'asset' }`).
+     * `{ source: 'parameter' }` lets a parameter-only change set roll the
+     * worker; see "Rolling the worker from CI" in docs/deploying-to-aws.md.
+     */
+    workerCode?: WorkerCode;
     /**
      * ADDITIONAL Secrets Manager ARNs the worker may read.
      *
@@ -115,6 +134,24 @@ export interface CanopyCmsServiceProps {
     environment?: Record<string, string>;
     /** EFS removal policy (default: RETAIN) */
     efsRemovalPolicy?: RemovalPolicy;
+    /**
+     * Daily AWS Backup of the EFS file system, kept 35 days (default true).
+     * Branches nobody has submitted yet exist only on EFS. Backup storage is
+     * billed per GB-month on top of EFS itself, and recovery points outlive the
+     * stack: their vault refuses deletes until its access policy is changed.
+     */
+    efsBackup?: boolean;
+    /**
+     * Give the file system a policy that refuses clients without TLS or without
+     * IAM authorization (default true). The worker and the Lambda mount with
+     * both, through the access point their roles are granted.
+     *
+     * A stack deployed before the worker mounted with IAM upgrades in two
+     * deploys, `false` first: CloudFormation updates the file system before it
+     * replaces the worker, so in one deploy the policy lands while the outgoing
+     * worker still holds an anonymous mount. See docs/adopter-migration.md.
+     */
+    efsEnforceIamAndTls?: boolean;
     /** GitHub owner for worker git operations (e.g., 'safeinsights') */
     githubOwner: string;
     /** GitHub repo name for worker git operations (e.g., 'docs-site') */
@@ -224,24 +261,26 @@ export interface CanopyCmsServiceProps {
      */
     clerkSecretKeySecretJsonField?: string;
     /**
-     * The GitHub repository's default branch name (default: 'main').
+     * The branch editing branches fork from (default: unset, so the worker
+     * detects it: the branch remote.git's HEAD names, or before remote.git exists
+     * GitHub's default branch).
      *
      * Interpolated straight into a git ref (`refs/heads/{baseBranch}`) and into
      * the worker's `.env` heredoc, so an invalid value is rejected at synth (see
      * `assertValidGitBranchName` above). A branch that git accepts but the repo
      * does not have fails permanently at boot instead: `verifyBaseBranchExists`
      * (packages/canopycms/src/worker/cms-worker.ts) throws, `worker/index.ts`
-     * exits 1, systemd's `Restart=always` repeats that forever, and
-     * `rebaseActiveBranches` rebases against the wrong lineage in the meantime.
+     * exits 1, and systemd's `Restart=always` repeats that forever.
      *
      * MUST match the shared repo's `canopycms.config.ts`'s `defaultBaseBranch`
-     * (both default to 'main' when unset) — the two are resolved by different
-     * processes (this stamps the worker's `.env`; the Lambda reads
-     * `config.defaultBaseBranch` at request time) with no automatic reconciliation
-     * between them. `infrastructure/lib/cms-stack.ts`, as scaffolded by
+     * whenever that is set. Leaving `defaultBaseBranch` unset is safe whether or
+     * not this is set: the worker points remote.git's HEAD at the branch it uses,
+     * and the Lambda reads that HEAD. Setting `defaultBaseBranch` alone is not:
+     * the two are resolved by different processes, and a Lambda configured with
+     * one name against a worker that detected another forks and rebases against
+     * different lineages. `infrastructure/lib/cms-stack.ts`, as scaffolded by
      * `canopycms init-deploy aws`, derives this prop FROM that config file so the
-     * two cannot drift; a hand-rolled stack must set it explicitly whenever
-     * `defaultBaseBranch` is anything other than 'main'.
+     * two cannot drift.
      */
     baseBranch?: string;
     /**
@@ -313,6 +352,18 @@ export interface CanopyCmsServiceProps {
      * default name would collide).
      */
     workerLogGroupName?: string;
+    /**
+     * SNS topic notified when the worker has stopped syncing git (and again when
+     * it recovers). Creating the topic and subscribing to it (email, Slack,
+     * PagerDuty) is the adopter's job. Default: none, and then no metric filter,
+     * alarm or alarm action is created.
+     *
+     * The alarm fires when the worker's log shows no git-sync cycle for 30
+     * minutes, which covers a crash loop, a boot loop, a worker that never
+     * started and a worker whose loops have stopped. A deploy or a spot
+     * replacement does not trip it. See `workerDownAlarm`.
+     */
+    alarmTopic?: sns.ITopic;
     /**
      * Execution role for the CMS Lambda (default: CDK creates one).
      *
@@ -429,6 +480,18 @@ export declare class CanopyCmsService extends Construct {
     readonly workerAsg: autoscaling.AutoScalingGroup;
     /** The EC2 worker's CloudWatch log group (worker stdout/stderr) */
     readonly workerLogGroup: logs.LogGroup;
+    /** Alarms when the worker logs no git-sync cycle for 30 minutes. Set only with `alarmTopic`. */
+    readonly workerDownAlarm?: cloudwatch.Alarm;
+    /**
+     * With `workerCode: { source: 'parameter' }`: the bucket worker bundles are
+     * uploaded to, as `canopy-worker/<sha256>.js`.
+     */
+    readonly workerBundleBucket?: s3.Bucket;
+    /**
+     * With `workerCode: { source: 'parameter' }`: the template parameter naming
+     * the bundle to run by its sha256; its logical id is also a stack output.
+     */
+    readonly workerBundleSha256Parameter?: CfnParameter;
     constructor(scope: Construct, id: string, props: CanopyCmsServiceProps);
     /**
      * Serve the editor from a CloudFront distribution you already own, such as

package/dist/constructs/cms-service.js

@@ -1,17 +1,9 @@
-import * as path from 'node:path';
-import { fileURLToPath } from 'node:url';
 import { Construct } from 'constructs';
-import { Annotations, ArnFormat, Duration, RemovalPolicy, Stack, Token, aws_ec2 as ec2, aws_efs as efs, aws_iam as iam, aws_lambda as lambda, aws_autoscaling as autoscaling, aws_s3_assets as s3assets, aws_logs as logs, } from 'aws-cdk-lib';
+import { Annotations, ArnFormat, Duration, Names, RemovalPolicy, Stack, Token, aws_cloudwatch as cloudwatch, aws_cloudwatch_actions as cloudwatchActions, aws_ec2 as ec2, aws_efs as efs, aws_iam as iam, aws_lambda as lambda, aws_autoscaling as autoscaling, aws_logs as logs, } from 'aws-cdk-lib';
 import { attachLambdaExecutionPolicies } from './lambda-execution-role.js';
+import { WORKER_BUNDLE_DOWNLOAD_PATH, workerBundleSource } from './worker-bundle.js';
 import { attachEditorBehaviors } from './editor-routing.js';
-import { EXIT_DRAINED_FOR_TERMINATION, WORKER_CAPACITY_ENV, WORKER_DRAIN_HOOK_NAME, } from './worker-lifecycle.js';
-// This package (`canopycms-cdk`) is `"type": "module"`, so its compiled output
-// is real ESM and `__dirname` is not a global there - the worker asset path
-// below throws `__dirname is not defined` under a real ESM runtime (e.g. `tsx`)
-// without this. Vitest's SSR/CJS-interop transform shims `__dirname`
-// automatically, so this file's own tests would not catch its absence. Same fix
-// as ../../lambda/asset-transform/build.mjs and ./asset-support.ts.
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
+import { EXIT_DRAINED_FOR_TERMINATION, WORKER_CAPACITY_ENV, WORKER_DRAIN_HOOK_NAME, WORKER_SYNC_LOG_PHRASE, } from './worker-lifecycle.js';
 /**
  * Synth-time mirror of `resolveDeploymentName`'s rule in the `canopycms`
  * package (packages/canopycms/src/operating-mode/deployment-name.ts).
@@ -45,6 +37,31 @@ const isValidDeploymentName = (name) => /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(name
  * by one process must resolve for the other.
  */
 const EFS_MOUNT_PATH = '/mnt/efs';
+/**
+ * The worker unit's sandbox, also in worker/canopy-worker.service. Node and git
+ * need none of what these take away; `MemoryDenyWriteExecute` is left out
+ * because V8's JIT writes executable memory.
+ */
+const WORKER_SANDBOX_DIRECTIVES = [
+    'NoNewPrivileges=yes',
+    'ProtectSystem=strict',
+    'ProtectHome=tmpfs',
+    'PrivateTmp=yes',
+    'PrivateDevices=yes',
+    'ProtectProc=invisible',
+    'ProtectKernelTunables=yes',
+    'ProtectKernelModules=yes',
+    'ProtectKernelLogs=yes',
+    'ProtectControlGroups=yes',
+    'ProtectClock=yes',
+    'ProtectHostname=yes',
+    'RestrictNamespaces=yes',
+    'RestrictSUIDSGID=yes',
+    'RestrictRealtime=yes',
+    'LockPersonality=yes',
+    'SystemCallArchitectures=native',
+    'CapabilityBoundingSet=',
+];
 /**
  * The heredoc delimiter user-data uses to write the worker's `.env` (see the
  * `cat > … << 'ENVEOF'` block below).
@@ -553,11 +570,14 @@ export class CanopyCmsService extends Construct {
         }
         // Both are interpolated into a git ref and the worker's `.env` heredoc, so
         // both are guarded at synth - see their doc comments for the failure each
-        // prevents. `settingsBranch` stays `undefined` (not stamped at all) unless
-        // the adopter explicitly set it: an absent env var and an empty one are NOT
-        // the same to the worker, which falls through to a computed name only when
+        // prevents. Each stays `undefined` (not stamped at all) unless the adopter
+        // explicitly set it, so the worker detects the base branch and computes the
+        // settings branch name. For the settings branch an absent env var and an
+        // empty one are NOT the same: the worker computes the name only when
         // `CANOPYCMS_SETTINGS_BRANCH` is unset entirely.
-        const baseBranch = assertValidGitBranchName('baseBranch', props.baseBranch ?? 'main');
+        const baseBranch = props.baseBranch !== undefined
+            ? assertValidGitBranchName('baseBranch', props.baseBranch)
+            : undefined;
         const settingsBranch = props.settingsBranch !== undefined
             ? assertValidGitBranchName('settingsBranch', props.settingsBranch)
             : undefined;
@@ -668,6 +688,19 @@ export class CanopyCmsService extends Construct {
             description: 'CanopyCMS EFS',
             allowAllOutbound: false,
         });
+        // With any policy in effect, EFS evaluates a client that mounts without
+        // `iam` against the Principal "*" statements alone, so a policy granting
+        // "*" nothing is what requires IAM: the worker and the Lambda get their
+        // access from their roles' grants (below, and CDK's for the Lambda's access
+        // point). Lambda uses TLS for every file-system connection
+        // (docs.aws.amazon.com/lambda/latest/dg/security-dataprotection.html). `allowAnonymousAccess: true` keeps CDK from adding its own
+        // "*" ClientWrite/ClientRootAccess statement, which it does under the
+        // `@aws-cdk/aws-efs:denyAnonymousAccess` flag or after a `grant*` call.
+        const efsClientActions = [
+            'elasticfilesystem:ClientMount',
+            'elasticfilesystem:ClientWrite',
+            'elasticfilesystem:ClientRootAccess',
+        ];
         this.fileSystem = new efs.FileSystem(this, 'FileSystem', {
             vpc: this.vpc,
             encrypted: true,
@@ -675,6 +708,20 @@ export class CanopyCmsService extends Construct {
             removalPolicy: props.efsRemovalPolicy ?? RemovalPolicy.RETAIN,
             vpcSubnets: { subnetType: ec2.SubnetType.PRIVATE_ISOLATED },
             securityGroup: efsSg,
+            enableAutomaticBackups: props.efsBackup ?? true,
+            allowAnonymousAccess: true,
+            fileSystemPolicy: (props.efsEnforceIamAndTls ?? true)
+                ? new iam.PolicyDocument({
+                    statements: [
+                        new iam.PolicyStatement({
+                            effect: iam.Effect.DENY,
+                            principals: [new iam.AnyPrincipal()],
+                            actions: efsClientActions,
+                            conditions: { Bool: { 'aws:SecureTransport': 'false' } },
+                        }),
+                    ],
+                })
+                : undefined,
         });
         const accessPoint = this.fileSystem.addAccessPoint('WorkspaceAP', {
             path: '/workspace',
@@ -870,8 +917,26 @@ export class CanopyCmsService extends Construct {
                 resources: secretsArns,
             }));
         }
-        // Worker needs EFS access (handled via security group, but mount needs ec2:DescribeAvailabilityZones)
-        workerRole.addManagedPolicy(iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonElasticFileSystemClientReadWriteAccess'));
+        // The worker's `iam` mount, through the shared access point only. The
+        // describe calls serve amazon-efs-utils' fallback to a mount target's IP
+        // address when the file system's DNS name does not resolve (it needs
+        // botocore too).
+        workerRole.addToPolicy(new iam.PolicyStatement({
+            actions: ['elasticfilesystem:ClientMount', 'elasticfilesystem:ClientWrite'],
+            resources: [this.fileSystem.fileSystemArn],
+            conditions: {
+                StringEquals: { 'elasticfilesystem:AccessPointArn': accessPoint.accessPointArn },
+            },
+        }));
+        workerRole.addToPolicy(new iam.PolicyStatement({
+            actions: ['elasticfilesystem:DescribeMountTargets'],
+            resources: [this.fileSystem.fileSystemArn],
+        }));
+        workerRole.addToPolicy(new iam.PolicyStatement({
+            // Supports no resource-level permissions.
+            actions: ['ec2:DescribeAvailabilityZones'],
+            resources: ['*'],
+        }));
         // Observation channel for a NAT-less deploy (SSM Session Manager / send-command);
         // worker SG already allows 443 egress.
         workerRole.addManagedPolicy(iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore'));
@@ -886,11 +951,48 @@ export class CanopyCmsService extends Construct {
         // CreateLogStream + PutLogEvents scoped to this group only (least privilege;
         // the group is pre-created by CFN so the agent never needs CreateLogGroup).
         this.workerLogGroup.grantWrite(workerRole);
-        // The worker is bundled with esbuild into a single JS file (pnpm run build:worker)
-        const workerAsset = new s3assets.Asset(this, 'WorkerCode', {
-            path: path.join(__dirname, '../../worker/dist'),
-        });
-        workerAsset.grantRead(workerRole);
+        if (props.alarmTopic) {
+            // Metrics are keyed by namespace and name alone, so the name carries this
+            // construct's unique id: two services in one account and region must not
+            // share one count. Capped at CloudWatch's 255 characters.
+            const metricPrefix = 'WorkerGitSyncCycles-';
+            const syncCycles = new logs.MetricFilter(this, 'WorkerSyncCycles', {
+                logGroup: this.workerLogGroup,
+                filterPattern: logs.FilterPattern.literal(`"${WORKER_SYNC_LOG_PHRASE}"`),
+                metricNamespace: 'CanopyCMS',
+                metricName: `${metricPrefix}${Names.uniqueResourceName(this, { maxLength: 255 - metricPrefix.length })}`,
+                metricValue: '1',
+            });
+            // Fires when 3 consecutive 10-minute periods hold no git-sync cycle line.
+            // The worker logs one every cycle (default 5 minutes), even while the
+            // schema gate holds the base branch. 30 minutes leaves room for a normal
+            // replacement: the terminating hook's drain
+            // (`workerTerminationHeartbeat`, 5 minutes by default), a boot of a few
+            // minutes plus its package upgrade, and the first sync. So a deploy or a
+            // spot replacement does not page, while a crash loop, a boot loop, a worker that never started and a
+            // worker whose loops have stopped all go silent and do. Missing data
+            // counts as a breach because a boot loop dies before the CloudWatch agent
+            // is installed and so writes no events at all. The construct does not
+            // expose CANOPYCMS_GIT_SYNC_INTERVAL; an adopter who sets it above about
+            // 10 minutes on the instance needs a wider window.
+            this.workerDownAlarm = new cloudwatch.Alarm(this, 'WorkerDownAlarm', {
+                metric: syncCycles.metric({ statistic: 'Sum', period: Duration.minutes(10) }),
+                threshold: 1,
+                comparisonOperator: cloudwatch.ComparisonOperator.LESS_THAN_THRESHOLD,
+                evaluationPeriods: 3,
+                datapointsToAlarm: 3,
+                treatMissingData: cloudwatch.TreatMissingData.BREACHING,
+                alarmDescription: `The CMS worker has not run a git sync for 30 minutes, so publishing and branch syncing ` +
+                    `are stalled. Check System health in the editor's admin panel and the worker log group ` +
+                    `${this.workerLogGroup.logGroupName}.`,
+            });
+            const topicAction = new cloudwatchActions.SnsAction(props.alarmTopic);
+            this.workerDownAlarm.addAlarmAction(topicAction);
+            this.workerDownAlarm.addOkAction(topicAction);
+        }
+        const workerBundle = workerBundleSource(this, props.workerCode ?? { source: 'asset' }, workerRole);
+        this.workerBundleBucket = workerBundle.bundleBucket;
+        this.workerBundleSha256Parameter = workerBundle.sha256Parameter;
         // Name/value PAIRS rather than pre-formatted lines: every value then flows
         // through `assertEnvSafe` in the single `map` below, so a value added here
         // later is guarded whether or not whoever adds it remembers to.
@@ -898,7 +1000,6 @@ export class CanopyCmsService extends Construct {
             ['CANOPYCMS_WORKSPACE_ROOT', EFS_MOUNT_PATH],
             ['CANOPYCMS_GITHUB_OWNER', props.githubOwner],
             ['CANOPYCMS_GITHUB_REPO', props.githubRepo],
-            ['CANOPYCMS_BASE_BRANCH', baseBranch],
             // The SAME string the Lambda's environment gets above, including an
             // `environment.CANOPYCMS_DEPLOYMENT_NAME` override - the two halves
             // resolve one settings branch (`canopycms-settings-<name>`) between them,
@@ -909,6 +1010,9 @@ export class CanopyCmsService extends Construct {
             // "Region is missing".
             ['AWS_REGION', Stack.of(this).region],
         ];
+        if (baseBranch !== undefined) {
+            envEntries.push(['CANOPYCMS_BASE_BRANCH', baseBranch]);
+        }
         if (props.githubTokenSecretArn) {
             envEntries.push(['CANOPYCMS_GITHUB_TOKEN_SECRET_ARN', props.githubTokenSecretArn]);
         }
@@ -963,9 +1067,9 @@ export class CanopyCmsService extends Construct {
         const envFileContent = envEntries
             .map(([name, value]) => `${name}=${assertEnvSafe(name, value)}`)
             .join('\n');
-        const efsMountOptions = `tls,accesspoint=${accessPoint.accessPointId}`;
+        const efsMountOptions = `tls,iam,accesspoint=${accessPoint.accessPointId}`;
         const userData = ec2.UserData.forLinux();
-        userData.addCommands('#!/bin/bash', 'set -euo pipefail', '', '# FAIL-FAST. Every step below is required for the worker to exist at', '# all, and until this trap existed a failure in any of them aborted', '# user-data BEFORE the systemd unit was written -- leaving an instance', "# that runs, passes the ASG's EC2-only health check indefinitely, and", '# does nothing. cfn-signal is deliberately not used here (it would', '# prove nothing about READINESS, which is the argument recorded below),', '# but that argument never covered a boot-SCRIPT failure: `cdk deploy`', '# reported success while publishes queued on EFS, the auth cache went', '# stale and PRs stopped being created, until a human noticed the admin', '# panel showing the worker absent.', '#', '# Shutting down makes the instance fail its EC2 health check, so the ASG', '# replaces it -- which is the only automatic recovery available in this', '# topology. The echo lands in the console log, readable via', '# `aws ec2 get-console-output`, because the CloudWatch agent is itself', '# configured further down and cannot be relied on to exist yet.', 'trap \'echo "canopy-worker user-data FAILED at line $LINENO (exit $?)" >&2; shutdown -h now\' ERR', '', '# Bounded retry for the network-dependent steps. Package mirrors and S3', '# have transient failures; a single flake should cost seconds, not an', '# instance replacement.', 'retry() {', '  local n=0', '  until "$@"; do', '    n=$((n + 1))', '    if [ "$n" -ge 5 ]; then', '      echo "canopy-worker: command failed after $n attempts: $*" >&2', '      return 1', '    fi', '    sleep $((n * 5))', '  done', '}', '', '# Install dependencies (unzip is not guaranteed in the AL2023 AMI)', 'retry dnf install -y git unzip', "# Node comes from AL2023's own repos, NOT a piped third-party installer.", '# The previous boot curled the NodeSource RPM setup script straight into', '# bash, so every instance replacement -- which the ASG performs on every', '# `cdk deploy`, plus every spot interruption -- depended on a third party', '# being reachable. cms-deploy.test.ts asserts that URL never comes back,', '# which is why it is not spelled out here.', '#', '# nodejs22, not 20: Node 20 reached upstream EOL on 2026-04-30, and this', '# repo declares `engines.node: ">=22"` with .nvmrc `v22`, so the worker', '# was running an EOL runtime BELOW the floor its own code is tested at.', "# 22 (EOL 2027-04-30) matches .nvmrc, CI, and the transform Lambda's", '# NODEJS_22_X, so one runtime is tested everywhere.', '#', '# ExecStart uses the NAMESPACED /usr/bin/node-22 (see the systemd unit', '# below), not the bare `node`: AL2023 installs versioned binaries and', '# points `/usr/bin/node` at one of them through `alternatives`, whose', '# selection AWS documents as able to change at any time.', 'retry dnf install -y nodejs22', '', '# Mount EFS through the same access point, at the same path, as the', '# Lambda (see EFS_MOUNT_PATH). amazon-efs-utils refuses an access-point', '# mount without tls (efs_utils_common/mount_options.py).', 'retry dnf install -y amazon-efs-utils', `mkdir -p ${EFS_MOUNT_PATH}`, `mount -t efs -o ${efsMountOptions} ${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH}`, '# Persist the mount across instance reboots: user-data runs once per', '# instance, so without an fstab entry a plain reboot leaves /mnt/efs an', '# empty local dir and the worker would clone a divergent remote.git', '# onto the instance disk, invisible to the Lambda.', `echo '${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH} efs _netdev,${efsMountOptions} 0 0' >> /etc/fstab`, '', '# Download worker from CDK S3 Asset', `retry aws s3 cp s3://${workerAsset.s3BucketName}/${workerAsset.s3ObjectKey} /tmp/canopy-worker.zip`, 'mkdir -p /opt/canopy-worker', 'cd /opt/canopy-worker', 'unzip -o /tmp/canopy-worker.zip', '# The worker bundle is ESM (esbuild --format=esm). Without this marker a', '# .js file is CommonJS by default and the import statement fails.', '# Node >=22.7 auto-detects module syntax and would mask its absence, and', '# this instance now runs node-22 -- so the marker is kept BECAUSE that', '# auto-detection is a fallback we should not depend on, not because the', '# runtime here still needs it.', `echo '{"type":"module"}' > /opt/canopy-worker/package.json`, '', '# Write environment file for systemd service', `cat > /opt/canopy-worker/.env << 'ENVEOF'`, envFileContent, 'ENVEOF', '', '# Create systemd service', `cat > /etc/systemd/system/canopy-worker.service << 'SVCEOF'`, '[Unit]', 'Description=CanopyCMS Worker Daemon', 'After=network.target', '# Never run against an unmounted /mnt/efs (see the fstab note above).', `RequiresMountsFor=${EFS_MOUNT_PATH}`, '', '[Service]', 'Type=simple', 'User=ec2-user', 'WorkingDirectory=/opt/canopy-worker', '# Namespaced binary, not bare `node`: AL2023 points /usr/bin/node at', '# an installed version through `alternatives`, and AWS documents that', '# selection as able to change at any time. node-22 always means 22.', 'ExecStart=/usr/bin/node-22 index.js', 'Restart=always', 'RestartSec=5', 'TimeoutStartSec=300', '# Stopping drains: SIGTERM reaches node ONLY (mixed), so the git', '# children it waits for keep running. Whatever is left when node', '# exits, or at TimeoutStopSec, is SIGKILLed; 120s outlasts the 90s', '# drain and its abort grace.', 'KillMode=mixed', 'TimeoutStopSec=120', '# A worker that drained for an instance termination exits with this;', '# restarting it would start work on an instance about to disappear.', `RestartPreventExitStatus=${EXIT_DRAINED_FOR_TERMINATION}`, `SuccessExitStatus=${EXIT_DRAINED_FOR_TERMINATION}`, '# File output, not journal: the CloudWatch agent cannot read journald,', '# so it tails this file instead (see the agent config below).', '# CAUTION: /var/log/canopy-worker must exist BEFORE first start. systemd', '# opens append: targets before it creates LogsDirectory= dirs', '# (systemd#27591), so without the pre-created dir the exec fails with', '# 209/STDOUT and Restart=always crash-loops forever. User-data runs', '# mkdir before systemctl start; LogsDirectory= is kept for ownership.', 'LogsDirectory=canopy-worker', 'StandardOutput=append:/var/log/canopy-worker/worker.log', 'StandardError=append:/var/log/canopy-worker/worker.log', 'EnvironmentFile=/opt/canopy-worker/.env', '', '[Install]', 'WantedBy=multi-user.target', 'SVCEOF', '', '# Set ownership for ec2-user', 'chown -R ec2-user:ec2-user /opt/canopy-worker', '', '# Pre-create the worker log dir (crash-loop guard, MUST precede the', '# first systemctl start): systemd opens StandardOutput=append: files', '# BEFORE it creates LogsDirectory= dirs (systemd#27591), so on a fresh', '# instance the unit would fail exec with 209/STDOUT and crash-loop', '# forever without this. The unit keeps LogsDirectory= for ownership', '# management on subsequent starts.', 'mkdir -p /var/log/canopy-worker', 'chown ec2-user:ec2-user /var/log/canopy-worker', '', '# Start worker', 'systemctl daemon-reload', 'systemctl enable canopy-worker', 'systemctl start canopy-worker', '', '# ---- CloudWatch log shipping ----', '# Placed AFTER worker start: with set -euo pipefail, a package/agent', '# failure here must not prevent the worker from running (shipping is', '# best-effort).', '#', '# DISARM THE FAIL-FAST TRAP AND errexit FIRST. Everything above this line', '# is required for the worker to exist at all, so a failure there should', '# replace the instance. Nothing below it is: the worker is already', '# running and healthy by this point. Leaving the trap armed would let a', '# package-mirror outage during the agent install shut down a perfectly', '# good worker -- and the ASG would relaunch straight into the same', '# outage, turning degraded log shipping into a replacement loop with the', '# worker down for its duration. That is strictly worse than the', '# best-effort behaviour this section has always documented.', 'trap - ERR', 'set +e', 'retry dnf install -y amazon-cloudwatch-agent logrotate', '', '# Bound on-disk growth; copytruncate keeps the fd the CW agent tails valid', '# (tiny copy->truncate loss window is acceptable for diagnostic logs).', `cat > /etc/logrotate.d/canopy-worker << 'ROTEOF'`, '/var/log/canopy-worker/worker.log {', '    size 10M', '    rotate 5', '    compress', '    copytruncate', '    missingok', '    notifempty', '}', 'ROTEOF', '# The config above only fires when logrotate actually runs; AL2023', '# presets may leave logrotate.timer disabled, and without it the size', '# cap never triggers and worker.log grows until the nano disk fills.', '# --now is idempotent if the timer is already enabled/running.', 'systemctl enable --now logrotate.timer', '', '# No retention_in_days here: CDK owns retention on the pre-created group.', `cat > /opt/aws/amazon-cloudwatch-agent/etc/canopy-worker-logs.json << 'CWEOF'`, '{', '  "logs": {', '    "logs_collected": {', '      "files": {', '        "collect_list": [', '          {', '            "file_path": "/var/log/canopy-worker/worker.log",', `            "log_group_name": "${this.workerLogGroup.logGroupName}",`, '            "log_stream_name": "{instance_id}",', 
+        userData.addCommands('#!/bin/bash', 'set -euo pipefail', '', '# FAIL-FAST. Every step below is required for the worker to exist at', '# all, and until this trap existed a failure in any of them aborted', '# user-data BEFORE the systemd unit was written -- leaving an instance', "# that runs, passes the ASG's EC2-only health check indefinitely, and", '# does nothing. cfn-signal is deliberately not used here (it would', '# prove nothing about READINESS, which is the argument recorded below),', '# but that argument never covered a boot-SCRIPT failure: `cdk deploy`', '# reported success while publishes queued on EFS, the auth cache went', '# stale and PRs stopped being created, until a human noticed the admin', '# panel showing the worker absent.', '#', '# Shutting down makes the instance fail its EC2 health check, so the ASG', '# replaces it -- which is the only automatic recovery available in this', '# topology. The echo lands in the console log, readable via', '# `aws ec2 get-console-output`, because the CloudWatch agent is itself', '# configured further down and cannot be relied on to exist yet.', 'trap \'echo "canopy-worker user-data FAILED at line $LINENO (exit $?)" >&2; shutdown -h now\' ERR', '', '# Bounded retry for the network-dependent steps. Package mirrors and S3', '# have transient failures; a single flake should cost seconds, not an', '# instance replacement.', 'retry() {', '  local n=0', '  until "$@"; do', '    n=$((n + 1))', '    if [ "$n" -ge 5 ]; then', '      echo "canopy-worker: command failed after $n attempts: $*" >&2', '      return 1', '    fi', '    sleep $((n * 5))', '  done', '}', '', '# Patch first. AL2023 locks dnf to the repository release its AMI was', '# built from, so a relaunch from the AMI resolved at the last deploy', '# would otherwise install nothing newer than that AMI. Moving to the', '# latest release also makes it the one every install below resolves', '# from. The kernel is left out: a new one applies only at a reboot,', '# which nothing here performs, so it would cost boot time and change', '# nothing. Kernel fixes arrive with the AMI a deploy resolves.', "retry dnf upgrade --releasever=latest --exclude='kernel*' -y", '', 'retry dnf install -y git', "# Node comes from AL2023's own repos, NOT a piped third-party installer.", '# The previous boot curled the NodeSource RPM setup script straight into', '# bash, so every instance replacement -- which the ASG performs on every', '# `cdk deploy`, plus every spot interruption -- depended on a third party', '# being reachable. cms-deploy.test.ts asserts that URL never comes back,', '# which is why it is not spelled out here.', '#', '# nodejs22, not 20: Node 20 reached upstream EOL on 2026-04-30, and this', '# repo declares `engines.node: ">=22"` with .nvmrc `v22`, so the worker', '# was running an EOL runtime BELOW the floor its own code is tested at.', "# 22 (EOL 2027-04-30) matches .nvmrc, CI, and the transform Lambda's", '# NODEJS_22_X, so one runtime is tested everywhere.', '#', '# ExecStart uses the NAMESPACED /usr/bin/node-22 (see the systemd unit', '# below), not the bare `node`: AL2023 installs versioned binaries and', '# points `/usr/bin/node` at one of them through `alternatives`, whose', '# selection AWS documents as able to change at any time.', 'retry dnf install -y nodejs22', '', '# Mount EFS through the same access point, at the same path, as the', '# Lambda (see EFS_MOUNT_PATH). amazon-efs-utils refuses an access-point', '# mount without tls (efs_utils_common/mount_options.py).', 'retry dnf install -y amazon-efs-utils', `mkdir -p ${EFS_MOUNT_PATH}`, '# Retried: an `iam` mount also fetches instance-role credentials.', `retry mount -t efs -o ${efsMountOptions} ${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH}`, '# Persist the mount across instance reboots: user-data runs once per', '# instance, so without an fstab entry a plain reboot leaves /mnt/efs an', '# empty local dir and the worker would clone a divergent remote.git', '# onto the instance disk, invisible to the Lambda.', `echo '${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH} efs _netdev,${efsMountOptions} 0 0' >> /etc/fstab`, '', '# The bundle runs only if its sha256 is the one given here (synthesized,', '# or the workerCode parameter); a mismatch fails the boot under the trap', '# above.', `retry aws s3 cp s3://${workerBundle.bucketName}/${workerBundle.objectKey} ${WORKER_BUNDLE_DOWNLOAD_PATH}`, `echo '${workerBundle.sha256}  ${WORKER_BUNDLE_DOWNLOAD_PATH}' | sha256sum -c -`, '# Root-owned, and read-only to the service (ProtectSystem=strict below).', `install -D -m 0644 ${WORKER_BUNDLE_DOWNLOAD_PATH} /opt/canopy-worker/index.js`, '# The worker bundle is ESM (esbuild --format=esm). Without this marker a', '# .js file is CommonJS by default and the import statement fails.', '# Node >=22.7 auto-detects module syntax and would mask its absence, and', '# this instance now runs node-22 -- so the marker is kept BECAUSE that', '# auto-detection is a fallback we should not depend on, not because the', '# runtime here still needs it.', `echo '{"type":"module"}' > /opt/canopy-worker/package.json`, '', '# Write environment file for systemd service', `cat > /opt/canopy-worker/.env << 'ENVEOF'`, envFileContent, 'ENVEOF', '', '# Create systemd service', `cat > /etc/systemd/system/canopy-worker.service << 'SVCEOF'`, '[Unit]', 'Description=CanopyCMS Worker Daemon', 'After=network.target', '# Never run against an unmounted /mnt/efs (see the fstab note above).', `RequiresMountsFor=${EFS_MOUNT_PATH}`, '', '[Service]', 'Type=simple', 'User=ec2-user', 'WorkingDirectory=/opt/canopy-worker', '# Namespaced binary, not bare `node`: AL2023 points /usr/bin/node at', '# an installed version through `alternatives`, and AWS documents that', '# selection as able to change at any time. node-22 always means 22.', 'ExecStart=/usr/bin/node-22 index.js', 'Restart=always', 'RestartSec=5', 'TimeoutStartSec=300', '# Stopping drains: SIGTERM reaches node ONLY (mixed), so the git', '# children it waits for keep running. Whatever is left when node', '# exits, or at TimeoutStopSec, is SIGKILLed; 120s outlasts the 90s', '# drain and its abort grace.', 'KillMode=mixed', 'TimeoutStopSec=120', '# A worker that drained for an instance termination exits with this;', '# restarting it would start work on an instance about to disappear.', `RestartPreventExitStatus=${EXIT_DRAINED_FOR_TERMINATION}`, `SuccessExitStatus=${EXIT_DRAINED_FOR_TERMINATION}`, '# File output, not journal: the CloudWatch agent cannot read journald,', '# so it tails this file instead (see the agent config below).', '# CAUTION: /var/log/canopy-worker must exist BEFORE first start. systemd', '# opens append: targets before it creates LogsDirectory= dirs', '# (systemd#27591), so without the pre-created dir the exec fails with', '# 209/STDOUT and Restart=always crash-loops forever. User-data runs', '# mkdir before systemctl start; LogsDirectory= is kept for ownership.', 'LogsDirectory=canopy-worker', 'StandardOutput=append:/var/log/canopy-worker/worker.log', 'StandardError=append:/var/log/canopy-worker/worker.log', '# The private GitHub mirror, /var/lib/canopy-worker on the root volume:', '# the only git repository the GitHub credential is used in. The worker', '# does not start without it.', 'StateDirectory=canopy-worker', 'EnvironmentFile=/opt/canopy-worker/.env', '# Sandbox. Writable: the workspace, the log dir (LogsDirectory=), the', '# state dir (StateDirectory=) and a private /tmp.', '# ProtectHome=tmpfs, not yes: every git call reads per-user files under', '# $HOME (~/.config/git/attributes, ignore), and an inaccessible /home', '# makes each read warn "Permission denied"; an empty one is a silent', '# ENOENT.', ...WORKER_SANDBOX_DIRECTIVES, `ReadWritePaths=${EFS_MOUNT_PATH}`, '', '[Install]', 'WantedBy=multi-user.target', 'SVCEOF', '', '# Pre-create the worker log dir (crash-loop guard, MUST precede the', '# first systemctl start): systemd opens StandardOutput=append: files', '# BEFORE it creates LogsDirectory= dirs (systemd#27591), so on a fresh', '# instance the unit would fail exec with 209/STDOUT and crash-loop', '# forever without this. The unit keeps LogsDirectory= for ownership', '# management on subsequent starts.', 'mkdir -p /var/log/canopy-worker', 'chown ec2-user:ec2-user /var/log/canopy-worker', '', '# Start worker', 'systemctl daemon-reload', 'systemctl enable canopy-worker', 'systemctl start canopy-worker', '', '# ---- CloudWatch log shipping ----', '# Placed AFTER worker start: with set -euo pipefail, a package/agent', '# failure here must not prevent the worker from running (shipping is', '# best-effort).', '#', '# DISARM THE FAIL-FAST TRAP AND errexit FIRST. Everything above this line', '# is required for the worker to exist at all, so a failure there should', '# replace the instance. Nothing below it is: the worker is already', '# running and healthy by this point. Leaving the trap armed would let a', '# package-mirror outage during the agent install shut down a perfectly', '# good worker -- and the ASG would relaunch straight into the same', '# outage, turning degraded log shipping into a replacement loop with the', '# worker down for its duration. That is strictly worse than the', '# best-effort behaviour this section has always documented.', 'trap - ERR', 'set +e', 'retry dnf install -y amazon-cloudwatch-agent logrotate', '', '# Bound on-disk growth; copytruncate keeps the fd the CW agent tails valid', '# (tiny copy->truncate loss window is acceptable for diagnostic logs).', `cat > /etc/logrotate.d/canopy-worker << 'ROTEOF'`, '/var/log/canopy-worker/worker.log {', '    size 10M', '    rotate 5', '    compress', '    copytruncate', '    missingok', '    notifempty', '}', 'ROTEOF', '# The config above only fires when logrotate actually runs; AL2023', '# presets may leave logrotate.timer disabled, and without it the size', '# cap never triggers and worker.log grows until the nano disk fills.', '# --now is idempotent if the timer is already enabled/running.', 'systemctl enable --now logrotate.timer', '', '# No retention_in_days here: CDK owns retention on the pre-created group.', `cat > /opt/aws/amazon-cloudwatch-agent/etc/canopy-worker-logs.json << 'CWEOF'`, '{', '  "logs": {', '    "logs_collected": {', '      "files": {', '        "collect_list": [', '          {', '            "file_path": "/var/log/canopy-worker/worker.log",', `            "log_group_name": "${this.workerLogGroup.logGroupName}",`, '            "log_stream_name": "{instance_id}",', 
         // The worker prefixes every line with an ISO-8601 timestamp
         // (packages/canopycms/src/worker/log.ts). Parsing it here is what makes
         // CloudWatch show the time the WORKER emitted a line rather than the
@@ -995,9 +1099,28 @@ export class CanopyCmsService extends Construct {
             role: workerRole,
             securityGroup: workerSg,
             userData,
+            requireImdsv2: true,
+            // 1, not AL2023's 2: everything that reads metadata runs on the host,
+            // and a second hop only serves containers.
+            httpPutResponseHopLimit: 1,
+            blockDevices: [
+                {
+                    // AL2023's root device. Its size is removed below.
+                    deviceName: '/dev/xvda',
+                    volume: ec2.BlockDeviceVolume.ebs(8, {
+                        encrypted: true,
+                        volumeType: ec2.EbsDeviceVolumeType.GP3,
+                        deleteOnTermination: true,
+                    }),
+                },
+            ],
         });
+        launchTemplate.node.defaultChild.addPropertyDeletionOverride('LaunchTemplateData.BlockDeviceMappings.0.Ebs.VolumeSize');
         this.workerAsg = new autoscaling.AutoScalingGroup(this, 'WorkerAsg', {
             vpc: this.vpc,
+            // Public, deliberately: it reaches GitHub and AWS APIs through the
+            // internet gateway instead of a NAT gateway that would cost more than
+            // the rest of the stack. Its security group admits no inbound traffic.
             vpcSubnets: { subnetType: ec2.SubnetType.PUBLIC },
             // Spot goes through a mixed-instances policy rather than spot options on
             // the launch template, so the group can choose among several pools.
@@ -1020,6 +1143,9 @@ export class CanopyCmsService extends Construct {
                 : { launchTemplate }),
             minCapacity: 1,
             maxCapacity: 1,
+            maxInstanceLifetime: props.workerMaxInstanceLifetime === null
+                ? undefined
+                : (props.workerMaxInstanceLifetime ?? Duration.days(7)),
             // `healthChecks`, not the deprecated `healthCheck`/`HealthCheck.ec2({ grace })`:
             // both synthesize the same HealthCheckType/HealthCheckGracePeriod, but the
             // deprecated form prints a jsii warning on every synth -- in adopters'
@@ -1030,9 +1156,9 @@ export class CanopyCmsService extends Construct {
             // Without an updatePolicy, CloudFormation's default behavior for an ASG
             // behind a changed launch template is to update the template resource
             // and do NOTHING else - the running instance keeps its old user-data
-            // (and therefore the old worker code: the worker bundle is a CDK S3
-            // asset whose hash is interpolated into user-data's `aws s3 cp
-            // s3://...`) until an interruption or a manual terminate happens to
+            // (and therefore the old worker code: the bundle's key and sha256,
+            // from the CDK asset or the workerCode parameter, are interpolated into
+            // user data) until an interruption or a manual terminate happens to
             // replace it. `cdk deploy` would then silently deploy everything
             // EXCEPT the worker. `rollingUpdate` makes CloudFormation actually
             // terminate-and-relaunch the instance on every deploy that changes the

package/dist/constructs/worker-bundle.d.ts

@@ -0,0 +1,36 @@
+import type { Construct } from 'constructs';
+import { CfnParameter, aws_iam as iam, aws_s3 as s3 } from 'aws-cdk-lib';
+/** Where user data downloads the bundle to and checks it, before installing it. */
+export declare const WORKER_BUNDLE_DOWNLOAD_PATH = "/tmp/canopy-worker.js";
+/**
+ * Where the worker's bundle comes from.
+ *
+ * - `'asset'` (default): a CDK asset, so a canopycms upgrade changes the
+ *   template and only a template deploy moves the worker.
+ * - `'parameter'`: a bucket of bundles keyed by sha256, chosen by a template
+ *   parameter, so a parameter-only change set (`--use-previous-template`)
+ *   can roll the worker. While the parameter is empty the worker runs the
+ *   template's own asset, as in `'asset'` mode. Once set, the parameter
+ *   outlives template deploys: `cdk deploy` keeps a parameter's previous
+ *   value.
+ */
+export type WorkerCode = {
+    source: 'asset';
+} | {
+    source: 'parameter';
+};
+interface WorkerBundleSource {
+    /** S3 location and expected sha256, as tokens for user data. */
+    readonly bucketName: string;
+    readonly objectKey: string;
+    readonly sha256: string;
+    /** Parameter mode only. */
+    readonly bundleBucket?: s3.Bucket;
+    readonly sha256Parameter?: CfnParameter;
+}
+/**
+ * The worker bundle's source, with the worker granted read on exactly what it
+ * may download.
+ */
+export declare function workerBundleSource(scope: Construct, workerCode: WorkerCode, workerRole: iam.IRole): WorkerBundleSource;
+export {};

package/dist/constructs/worker-bundle.js

@@ -0,0 +1,87 @@
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+import * as path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { CfnCondition, CfnOutput, CfnParameter, Duration, Fn, RemovalPolicy, Token, aws_iam as iam, aws_s3 as s3, aws_s3_assets as s3assets, } from 'aws-cdk-lib';
+// This package is `"type": "module"`, so `__dirname` is not a global in its
+// compiled output. Vitest shims it, so this file's tests would not notice.
+// Same fix as ./asset-support.ts.
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+/**
+ * The worker bundle: the single file esbuild writes (`pnpm run build:worker`),
+ * shipped to the instance as-is so its sha256 is known at synth. A directory
+ * asset would be zipped by cdk-assets at publish time, after synth.
+ */
+const WORKER_BUNDLE_PATH = path.join(__dirname, '../../worker/dist/index.js');
+/** Where user data downloads the bundle to and checks it, before installing it. */
+export const WORKER_BUNDLE_DOWNLOAD_PATH = '/tmp/canopy-worker.js';
+/** Where `workerCode: { source: 'parameter' }` expects bundles in its bucket: `<prefix><sha256>.js`. */
+const WORKER_BUNDLE_KEY_PREFIX = 'canopy-worker/';
+function sha256OfFile(file) {
+    return createHash('sha256').update(readFileSync(file)).digest('hex');
+}
+/**
+ * The worker bundle's source, with the worker granted read on exactly what it
+ * may download.
+ */
+export function workerBundleSource(scope, workerCode, workerRole) {
+    const asset = new s3assets.Asset(scope, 'WorkerCode', { path: WORKER_BUNDLE_PATH });
+    const assetSha256 = sha256OfFile(WORKER_BUNDLE_PATH);
+    // This one object, not `asset.grantRead`'s whole bootstrap bucket. No KMS
+    // grant: the bootstrap bucket's key (aws/s3, or the key bootstrap creates)
+    // lets any principal in the account decrypt through S3.
+    const readable = [asset.bucket.arnForObjects(asset.s3ObjectKey)];
+    if (workerCode.source === 'asset') {
+        workerRole.addToPrincipalPolicy(new iam.PolicyStatement({ actions: ['s3:GetObject'], resources: readable }));
+        return { bucketName: asset.s3BucketName, objectKey: asset.s3ObjectKey, sha256: assetSha256 };
+    }
+    const bundleBucket = new s3.Bucket(scope, 'WorkerBundleBucket', {
+        versioned: true,
+        blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL,
+        encryption: s3.BucketEncryption.S3_MANAGED,
+        enforceSSL: true,
+        removalPolicy: RemovalPolicy.RETAIN,
+        // Re-uploading a bundle under its own hash leaves a noncurrent version of
+        // the same bytes; a launch template only ever names current ones.
+        lifecycleRules: [{ noncurrentVersionExpiration: Duration.days(30) }],
+    });
+    // A bundle a running or rolled-back launch template names must stay
+    // downloadable. Not PutBucketPolicy too: that would lock CloudFormation out
+    // of this policy. Emptying the bucket starts by removing this statement.
+    bundleBucket.addToResourcePolicy(new iam.PolicyStatement({
+        effect: iam.Effect.DENY,
+        principals: [new iam.AnyPrincipal()],
+        actions: ['s3:DeleteObject', 's3:DeleteObjectVersion'],
+        resources: [bundleBucket.arnForObjects('*')],
+    }));
+    readable.push(bundleBucket.arnForObjects(`${WORKER_BUNDLE_KEY_PREFIX}*`));
+    workerRole.addToPrincipalPolicy(new iam.PolicyStatement({ actions: ['s3:GetObject'], resources: readable }));
+    // One value names the object AND is what it is checked against, so a key
+    // and a hash cannot disagree.
+    const sha256Parameter = new CfnParameter(scope, 'WorkerBundleSha256', {
+        type: 'String',
+        default: '',
+        allowedPattern: '^([0-9a-f]{64})?$',
+        description: `sha256 of the CanopyCMS worker bundle to run, uploaded to the worker bundle bucket as ` +
+            `${WORKER_BUNDLE_KEY_PREFIX}<sha256>.js. Empty runs the bundle this template was synthesized with.`,
+    });
+    const fromParameter = new CfnCondition(scope, 'WorkerBundleFromParameter', {
+        expression: Fn.conditionNot(Fn.conditionEquals(sha256Parameter.valueAsString, '')),
+    });
+    const pick = (whenParameter, whenAsset) => Token.asString(Fn.conditionIf(fromParameter.logicalId, whenParameter, whenAsset));
+    new CfnOutput(scope, 'WorkerBundleSha256ParameterName', {
+        description: 'The template parameter that selects the worker bundle',
+        value: sha256Parameter.logicalId,
+    });
+    new CfnOutput(scope, 'WorkerBundleBucketName', {
+        description: `The bucket holding worker bundles, as ${WORKER_BUNDLE_KEY_PREFIX}<sha256>.js`,
+        value: bundleBucket.bucketName,
+    });
+    return {
+        bucketName: pick(bundleBucket.bucketName, asset.s3BucketName),
+        objectKey: pick(Fn.join('', [WORKER_BUNDLE_KEY_PREFIX, sha256Parameter.valueAsString, '.js']), asset.s3ObjectKey),
+        sha256: pick(sha256Parameter.valueAsString, assetSha256),
+        bundleBucket,
+        sha256Parameter,
+    };
+}

package/dist/constructs/worker-lifecycle.d.ts

@@ -11,8 +11,22 @@ export declare const WORKER_DRAIN_HOOK_NAME = "canopycms-worker-drain";
  * does not start a fresh worker on an instance about to disappear.
  */
 export declare const EXIT_DRAINED_FOR_TERMINATION = 75;
+/**
+ * The worker's exit status after it stopped itself (its EFS lock was
+ * compromised). Non-zero, and deliberately not {@link EXIT_DRAINED_FOR_TERMINATION}
+ * (75), which the unit lists in `RestartPreventExitStatus=`: this one must let
+ * `Restart=always` start a fresh worker. Not in `SuccessExitStatus=` either.
+ * 69 is sysexits' EX_UNAVAILABLE.
+ */
+export declare const EXIT_WORKER_SELF_STOPPED = 69;
 /**
  * Set to `spot` in the worker's environment when it runs on spot capacity, which
  * arms the watch for a spot interruption notice.
  */
 export declare const WORKER_CAPACITY_ENV = "CANOPYCMS_WORKER_CAPACITY";
+/**
+ * The text of the line `syncGit()` (canopycms `worker/git-sync.ts`) logs at the
+ * start of every git-sync cycle. The worker-down alarm's metric filter counts
+ * it, so the alarm misfires if the worker's text drifts from this one.
+ */
+export declare const WORKER_SYNC_LOG_PHRASE = "Syncing git...";