canopycms-cdk 0.0.68-int.111 → 0.0.68-int.113

package/dist/constructs/cms-service.d.ts

@@ -362,6 +362,9 @@ export interface CanopyCmsServiceProps {
      * minutes, which covers a crash loop, a boot loop, a worker that never
      * started and a worker whose loops have stopped. A deploy or a spot
      * replacement does not trip it. See `workerDownAlarm`.
+     *
+     * The topic is also notified when a worker boot could not upgrade its
+     * packages and started on the AMI's. See `workerUnpatchedAlarm`.
      */
     alarmTopic?: sns.ITopic;
     /**
@@ -482,6 +485,8 @@ export declare class CanopyCmsService extends Construct {
     readonly workerLogGroup: logs.LogGroup;
     /** Alarms when the worker logs no git-sync cycle for 30 minutes. Set only with `alarmTopic`. */
     readonly workerDownAlarm?: cloudwatch.Alarm;
+    /** Alarms when a worker boot started without its package upgrade. Set only with `alarmTopic`. */
+    readonly workerUnpatchedAlarm?: cloudwatch.Alarm;
     /**
      * With `workerCode: { source: 'parameter' }`: the bucket worker bundles are
      * uploaded to, as `canopy-worker/<sha256>.js`.

package/dist/constructs/cms-service.js

@@ -3,7 +3,7 @@ import { Annotations, ArnFormat, Duration, Names, RemovalPolicy, Stack, Token, a
 import { attachLambdaExecutionPolicies } from './lambda-execution-role.js';
 import { WORKER_BUNDLE_DOWNLOAD_PATH, workerBundleSource } from './worker-bundle.js';
 import { attachEditorBehaviors } from './editor-routing.js';
-import { EXIT_DRAINED_FOR_TERMINATION, WORKER_CAPACITY_ENV, WORKER_DRAIN_HOOK_NAME, WORKER_SYNC_LOG_PHRASE, } from './worker-lifecycle.js';
+import { EXIT_DRAINED_FOR_TERMINATION, WORKER_CAPACITY_ENV, WORKER_CONTRACT_ENV, WORKER_CONTRACT_VERSION, WORKER_DRAIN_HOOK_NAME, WORKER_SYNC_LOG_PHRASE, } from './worker-lifecycle.js';
 /**
  * Synth-time mirror of `resolveDeploymentName`'s rule in the `canopycms`
  * package (packages/canopycms/src/operating-mode/deployment-name.ts).
@@ -37,6 +37,12 @@ const isValidDeploymentName = (name) => /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(name
  * by one process must resolve for the other.
  */
 const EFS_MOUNT_PATH = '/mnt/efs';
+/**
+ * The text of the worker.log line user data writes when the boot's package
+ * upgrade failed and the worker runs on the AMI's packages. The unpatched
+ * alarm's metric filter counts it.
+ */
+const WORKER_UNPATCHED_LOG_PHRASE = 'running unpatched on the AMI packages until the next boot';
 /**
  * The worker unit's sandbox, also in worker/canopy-worker.service. Node and git
  * need none of what these take away; `MemoryDenyWriteExecute` is left out
@@ -989,6 +995,28 @@ export class CanopyCmsService extends Construct {
             const topicAction = new cloudwatchActions.SnsAction(props.alarmTopic);
             this.workerDownAlarm.addAlarmAction(topicAction);
             this.workerDownAlarm.addOkAction(topicAction);
+            const unpatchedPrefix = 'WorkerUnpatchedBoots-';
+            const unpatchedBoots = new logs.MetricFilter(this, 'WorkerUnpatchedBoots', {
+                logGroup: this.workerLogGroup,
+                filterPattern: logs.FilterPattern.literal(`"${WORKER_UNPATCHED_LOG_PHRASE}"`),
+                metricNamespace: 'CanopyCMS',
+                metricName: `${unpatchedPrefix}${Names.uniqueResourceName(this, { maxLength: 255 - unpatchedPrefix.length })}`,
+                metricValue: '1',
+            });
+            // One line per unpatched boot, so the alarm returns to OK one period
+            // later on its own; an OK notification would only repeat the alarm.
+            this.workerUnpatchedAlarm = new cloudwatch.Alarm(this, 'WorkerUnpatchedAlarm', {
+                metric: unpatchedBoots.metric({ statistic: 'Sum', period: Duration.minutes(10) }),
+                threshold: 1,
+                comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
+                evaluationPeriods: 1,
+                treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
+                alarmDescription: `A CMS worker boot could not upgrade its packages and started on the AMI's. It is ` +
+                    `working, and the next boot retries the upgrade. The boot's console output ` +
+                    `(aws ec2 get-console-output) and the worker log group ` +
+                    `${this.workerLogGroup.logGroupName} name the failure.`,
+            });
+            this.workerUnpatchedAlarm.addAlarmAction(topicAction);
         }
         const workerBundle = workerBundleSource(this, props.workerCode ?? { source: 'asset' }, workerRole);
         this.workerBundleBucket = workerBundle.bundleBucket;
@@ -1069,7 +1097,7 @@ export class CanopyCmsService extends Construct {
             .join('\n');
         const efsMountOptions = `tls,iam,accesspoint=${accessPoint.accessPointId}`;
         const userData = ec2.UserData.forLinux();
-        userData.addCommands('#!/bin/bash', 'set -euo pipefail', '', '# FAIL-FAST. Every step below is required for the worker to exist at', '# all, and until this trap existed a failure in any of them aborted', '# user-data BEFORE the systemd unit was written -- leaving an instance', "# that runs, passes the ASG's EC2-only health check indefinitely, and", '# does nothing. cfn-signal is deliberately not used here (it would', '# prove nothing about READINESS, which is the argument recorded below),', '# but that argument never covered a boot-SCRIPT failure: `cdk deploy`', '# reported success while publishes queued on EFS, the auth cache went', '# stale and PRs stopped being created, until a human noticed the admin', '# panel showing the worker absent.', '#', '# Shutting down makes the instance fail its EC2 health check, so the ASG', '# replaces it -- which is the only automatic recovery available in this', '# topology. The echo lands in the console log, readable via', '# `aws ec2 get-console-output`, because the CloudWatch agent is itself', '# configured further down and cannot be relied on to exist yet.', 'trap \'echo "canopy-worker user-data FAILED at line $LINENO (exit $?)" >&2; shutdown -h now\' ERR', '', '# Bounded retry for the network-dependent steps. Package mirrors and S3', '# have transient failures; a single flake should cost seconds, not an', '# instance replacement.', 'retry() {', '  local n=0', '  until "$@"; do', '    n=$((n + 1))', '    if [ "$n" -ge 5 ]; then', '      echo "canopy-worker: command failed after $n attempts: $*" >&2', '      return 1', '    fi', '    sleep $((n * 5))', '  done', '}', '', '# Patch first. AL2023 locks dnf to the repository release its AMI was', '# built from, so a relaunch from the AMI resolved at the last deploy', '# would otherwise install nothing newer than that AMI. Moving to the', '# latest release also makes it the one every install below resolves', '# from. The kernel is left out: a new one applies only at a reboot,', '# which nothing here performs, so it would cost boot time and change', '# nothing. Kernel fixes arrive with the AMI a deploy resolves.', "retry dnf upgrade --releasever=latest --exclude='kernel*' -y", '', 'retry dnf install -y git', "# Node comes from AL2023's own repos, NOT a piped third-party installer.", '# The previous boot curled the NodeSource RPM setup script straight into', '# bash, so every instance replacement -- which the ASG performs on every', '# `cdk deploy`, plus every spot interruption -- depended on a third party', '# being reachable. cms-deploy.test.ts asserts that URL never comes back,', '# which is why it is not spelled out here.', '#', '# nodejs22, not 20: Node 20 reached upstream EOL on 2026-04-30, and this', '# repo declares `engines.node: ">=22"` with .nvmrc `v22`, so the worker', '# was running an EOL runtime BELOW the floor its own code is tested at.', "# 22 (EOL 2027-04-30) matches .nvmrc, CI, and the transform Lambda's", '# NODEJS_22_X, so one runtime is tested everywhere.', '#', '# ExecStart uses the NAMESPACED /usr/bin/node-22 (see the systemd unit', '# below), not the bare `node`: AL2023 installs versioned binaries and', '# points `/usr/bin/node` at one of them through `alternatives`, whose', '# selection AWS documents as able to change at any time.', 'retry dnf install -y nodejs22', '', '# Mount EFS through the same access point, at the same path, as the', '# Lambda (see EFS_MOUNT_PATH). amazon-efs-utils refuses an access-point', '# mount without tls (efs_utils_common/mount_options.py).', 'retry dnf install -y amazon-efs-utils', `mkdir -p ${EFS_MOUNT_PATH}`, '# Retried: an `iam` mount also fetches instance-role credentials.', `retry mount -t efs -o ${efsMountOptions} ${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH}`, '# Persist the mount across instance reboots: user-data runs once per', '# instance, so without an fstab entry a plain reboot leaves /mnt/efs an', '# empty local dir and the worker would clone a divergent remote.git', '# onto the instance disk, invisible to the Lambda.', `echo '${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH} efs _netdev,${efsMountOptions} 0 0' >> /etc/fstab`, '', '# The bundle runs only if its sha256 is the one given here (synthesized,', '# or the workerCode parameter); a mismatch fails the boot under the trap', '# above.', `retry aws s3 cp s3://${workerBundle.bucketName}/${workerBundle.objectKey} ${WORKER_BUNDLE_DOWNLOAD_PATH}`, `echo '${workerBundle.sha256}  ${WORKER_BUNDLE_DOWNLOAD_PATH}' | sha256sum -c -`, '# Root-owned, and read-only to the service (ProtectSystem=strict below).', `install -D -m 0644 ${WORKER_BUNDLE_DOWNLOAD_PATH} /opt/canopy-worker/index.js`, '# The worker bundle is ESM (esbuild --format=esm). Without this marker a', '# .js file is CommonJS by default and the import statement fails.', '# Node >=22.7 auto-detects module syntax and would mask its absence, and', '# this instance now runs node-22 -- so the marker is kept BECAUSE that', '# auto-detection is a fallback we should not depend on, not because the', '# runtime here still needs it.', `echo '{"type":"module"}' > /opt/canopy-worker/package.json`, '', '# Write environment file for systemd service', `cat > /opt/canopy-worker/.env << 'ENVEOF'`, envFileContent, 'ENVEOF', '', '# Create systemd service', `cat > /etc/systemd/system/canopy-worker.service << 'SVCEOF'`, '[Unit]', 'Description=CanopyCMS Worker Daemon', 'After=network.target', '# Never run against an unmounted /mnt/efs (see the fstab note above).', `RequiresMountsFor=${EFS_MOUNT_PATH}`, '', '[Service]', 'Type=simple', 'User=ec2-user', 'WorkingDirectory=/opt/canopy-worker', '# Namespaced binary, not bare `node`: AL2023 points /usr/bin/node at', '# an installed version through `alternatives`, and AWS documents that', '# selection as able to change at any time. node-22 always means 22.', 'ExecStart=/usr/bin/node-22 index.js', 'Restart=always', 'RestartSec=5', 'TimeoutStartSec=300', '# Stopping drains: SIGTERM reaches node ONLY (mixed), so the git', '# children it waits for keep running. Whatever is left when node', '# exits, or at TimeoutStopSec, is SIGKILLed; 120s outlasts the 90s', '# drain and its abort grace.', 'KillMode=mixed', 'TimeoutStopSec=120', '# A worker that drained for an instance termination exits with this;', '# restarting it would start work on an instance about to disappear.', `RestartPreventExitStatus=${EXIT_DRAINED_FOR_TERMINATION}`, `SuccessExitStatus=${EXIT_DRAINED_FOR_TERMINATION}`, '# File output, not journal: the CloudWatch agent cannot read journald,', '# so it tails this file instead (see the agent config below).', '# CAUTION: /var/log/canopy-worker must exist BEFORE first start. systemd', '# opens append: targets before it creates LogsDirectory= dirs', '# (systemd#27591), so without the pre-created dir the exec fails with', '# 209/STDOUT and Restart=always crash-loops forever. User-data runs', '# mkdir before systemctl start; LogsDirectory= is kept for ownership.', 'LogsDirectory=canopy-worker', 'StandardOutput=append:/var/log/canopy-worker/worker.log', 'StandardError=append:/var/log/canopy-worker/worker.log', '# The private GitHub mirror, /var/lib/canopy-worker on the root volume:', '# the only git repository the GitHub credential is used in. The worker', '# does not start without it.', 'StateDirectory=canopy-worker', 'EnvironmentFile=/opt/canopy-worker/.env', '# Sandbox. Writable: the workspace, the log dir (LogsDirectory=), the', '# state dir (StateDirectory=) and a private /tmp.', '# ProtectHome=tmpfs, not yes: every git call reads per-user files under', '# $HOME (~/.config/git/attributes, ignore), and an inaccessible /home', '# makes each read warn "Permission denied"; an empty one is a silent', '# ENOENT.', ...WORKER_SANDBOX_DIRECTIVES, `ReadWritePaths=${EFS_MOUNT_PATH}`, '', '[Install]', 'WantedBy=multi-user.target', 'SVCEOF', '', '# Pre-create the worker log dir (crash-loop guard, MUST precede the', '# first systemctl start): systemd opens StandardOutput=append: files', '# BEFORE it creates LogsDirectory= dirs (systemd#27591), so on a fresh', '# instance the unit would fail exec with 209/STDOUT and crash-loop', '# forever without this. The unit keeps LogsDirectory= for ownership', '# management on subsequent starts.', 'mkdir -p /var/log/canopy-worker', 'chown ec2-user:ec2-user /var/log/canopy-worker', '', '# Start worker', 'systemctl daemon-reload', 'systemctl enable canopy-worker', 'systemctl start canopy-worker', '', '# ---- CloudWatch log shipping ----', '# Placed AFTER worker start: with set -euo pipefail, a package/agent', '# failure here must not prevent the worker from running (shipping is', '# best-effort).', '#', '# DISARM THE FAIL-FAST TRAP AND errexit FIRST. Everything above this line', '# is required for the worker to exist at all, so a failure there should', '# replace the instance. Nothing below it is: the worker is already', '# running and healthy by this point. Leaving the trap armed would let a', '# package-mirror outage during the agent install shut down a perfectly', '# good worker -- and the ASG would relaunch straight into the same', '# outage, turning degraded log shipping into a replacement loop with the', '# worker down for its duration. That is strictly worse than the', '# best-effort behaviour this section has always documented.', 'trap - ERR', 'set +e', 'retry dnf install -y amazon-cloudwatch-agent logrotate', '', '# Bound on-disk growth; copytruncate keeps the fd the CW agent tails valid', '# (tiny copy->truncate loss window is acceptable for diagnostic logs).', `cat > /etc/logrotate.d/canopy-worker << 'ROTEOF'`, '/var/log/canopy-worker/worker.log {', '    size 10M', '    rotate 5', '    compress', '    copytruncate', '    missingok', '    notifempty', '}', 'ROTEOF', '# The config above only fires when logrotate actually runs; AL2023', '# presets may leave logrotate.timer disabled, and without it the size', '# cap never triggers and worker.log grows until the nano disk fills.', '# --now is idempotent if the timer is already enabled/running.', 'systemctl enable --now logrotate.timer', '', '# No retention_in_days here: CDK owns retention on the pre-created group.', `cat > /opt/aws/amazon-cloudwatch-agent/etc/canopy-worker-logs.json << 'CWEOF'`, '{', '  "logs": {', '    "logs_collected": {', '      "files": {', '        "collect_list": [', '          {', '            "file_path": "/var/log/canopy-worker/worker.log",', `            "log_group_name": "${this.workerLogGroup.logGroupName}",`, '            "log_stream_name": "{instance_id}",', 
+        userData.addCommands('#!/bin/bash', 'set -euo pipefail', '', '# FAIL-FAST. Every step below is required for the worker to exist at', '# all, except the swap and the package upgrade, which are written as', '# conditions so that they cannot trip it. Until this trap existed, a', '# failure in any required step aborted user-data BEFORE the systemd unit was written -- leaving an instance', "# that runs, passes the ASG's EC2-only health check indefinitely, and", '# does nothing. cfn-signal is deliberately not used here (it would', '# prove nothing about READINESS, which is the argument recorded below),', '# but that argument never covered a boot-SCRIPT failure: `cdk deploy`', '# reported success while publishes queued on EFS, the auth cache went', '# stale and PRs stopped being created, until a human noticed the admin', '# panel showing the worker absent.', '#', '# Shutting down makes the instance fail its EC2 health check, so the ASG', '# replaces it -- which is the only automatic recovery available in this', '# topology. The echo lands in the console log, readable via', '# `aws ec2 get-console-output`, because the CloudWatch agent is itself', '# configured further down and cannot be relied on to exist yet.', 'trap \'echo "canopy-worker user-data FAILED at line $LINENO (exit $?)" >&2; shutdown -h now\' ERR', '', '# Bounded retry for the network-dependent steps. Package mirrors and S3', '# have transient failures; a single flake should cost seconds, not an', '# instance replacement. The first argument names the step in the message', '# an exhausted step leaves on the console.', 'retry() {', '  local step=$1 n=0', '  shift', '  until "$@"; do', '    n=$((n + 1))', '    if [ "$n" -ge 5 ]; then', `      echo "canopy-worker boot: '$step' failed after $n attempts" >&2`, '      return 1', '    fi', '    sleep $((n * 5))', '  done', '}', '', '# Swap, before the first dnf: loading the repository metadata takes more', '# memory than a t4g.nano has free after boot, and the kernel OOM-kills', '# dnf. It stays on afterwards at low swappiness, as headroom for git and', '# the CloudWatch agent rather than working memory. It is on the encrypted', '# root volume, so memory paged out of the worker is encrypted at rest.', '# fallocate is safe here: XFS, the root file system, supports', '# preallocated swap files. A failure only warns, because dnf may still fit', '# and a boot that stopped here would put the group into a replacement', '# loop. Each command carries `|| return 1` because errexit does not apply', '# inside a function called as a condition.', 'setup_swap() {', '  swapon --show=NAME --noheadings | grep -qx /swapfile && return 0', '  if [ "$(stat -c %s /swapfile 2>/dev/null)" != 1073741824 ]; then', '    # The file, plus 2 GiB for packages, logs and the GitHub mirror.', '    local avail_mib', `    avail_mib=$(df --output=avail -m / | tail -n 1 | tr -d ' ')`, '    case "$avail_mib" in ""|*[!0-9]*) return 1 ;; esac', '    if [ "$avail_mib" -lt 3072 ]; then', '      echo "canopy-worker boot: under 3 GiB free on /, so no swap file" >&2', '      return 1', '    fi', '    rm -f /swapfile || return 1', '    fallocate -l 1G /swapfile || return 1', '  fi', '  chmod 600 /swapfile || return 1', '  mkswap /swapfile || return 1', '  # swapon last, so that swap is never on when this returns 1.', "  grep -qs '^/swapfile ' /etc/fstab || echo '/swapfile none swap defaults 0 0' >> /etc/fstab || return 1", '  mkdir -p /etc/sysctl.d || return 1', "  echo 'vm.swappiness = 10' > /etc/sysctl.d/90-canopy-worker-swap.conf || return 1", '  sysctl -q -w vm.swappiness=10 || return 1', '  swapon /swapfile', '}', 'if ! setup_swap; then', `  echo "canopy-worker boot: 'swap' setup failed; continuing without swap" >&2`, 'fi', '', '# Patch first. AL2023 locks dnf to the repository release its AMI was', '# built from, so a relaunch from the AMI resolved at the last deploy', '# would otherwise install nothing newer than that AMI. Moving to the', '# latest release also makes it the one every install below resolves', '# from. The kernel is left out: a new one applies only at a reboot,', '# which nothing here performs, so it would cost boot time and change', '# nothing. Kernel fixes arrive with the AMI a deploy resolves.', '#', "# A failed upgrade does not stop the boot: the worker starts on the AMI's", '# packages and the next boot (by default the weekly recycle at the', '# latest) tries again. Failing here would put the group into a', '# replacement loop with no worker at all. The installs below then', '# resolve from the AMI release: 2023.7.20250428 carries all of them,', '# 2023.7.20250331 lacks nodejs22. The failure is written to worker.log', '# below, for the unpatched alarm.', 'WORKER_UNPATCHED=0', "if ! retry 'dnf upgrade' dnf upgrade --releasever=latest --exclude='kernel*' -y; then", '  WORKER_UNPATCHED=1', `  echo "canopy-worker boot: ${WORKER_UNPATCHED_LOG_PHRASE}" >&2`, 'fi', '', "retry 'dnf install git' dnf install -y git", "# Node comes from AL2023's own repos, NOT a piped third-party installer.", '# The previous boot curled the NodeSource RPM setup script straight into', '# bash, so every instance replacement -- which the ASG performs on every', '# `cdk deploy`, plus every spot interruption -- depended on a third party', '# being reachable. cms-deploy.test.ts asserts that URL never comes back,', '# which is why it is not spelled out here.', '#', '# nodejs22, not 20: Node 20 reached upstream EOL on 2026-04-30, and this', '# repo declares `engines.node: ">=22"` with .nvmrc `v22`, so the worker', '# was running an EOL runtime BELOW the floor its own code is tested at.', "# 22 (EOL 2027-04-30) matches .nvmrc, CI, and the transform Lambda's", '# NODEJS_22_X, so one runtime is tested everywhere.', '#', '# ExecStart uses the NAMESPACED /usr/bin/node-22 (see the systemd unit', '# below), not the bare `node`: AL2023 installs versioned binaries and', '# points `/usr/bin/node` at one of them through `alternatives`, whose', '# selection AWS documents as able to change at any time.', "retry 'dnf install nodejs22' dnf install -y nodejs22", '', '# Mount EFS through the same access point, at the same path, as the', '# Lambda (see EFS_MOUNT_PATH). amazon-efs-utils refuses an access-point', '# mount without tls (efs_utils_common/mount_options.py).', "retry 'dnf install amazon-efs-utils' dnf install -y amazon-efs-utils", `mkdir -p ${EFS_MOUNT_PATH}`, '# Retried: an `iam` mount also fetches instance-role credentials.', `retry 'EFS mount' mount -t efs -o ${efsMountOptions} ${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH}`, '# Persist the mount across instance reboots: user-data runs once per', '# instance, so without an fstab entry a plain reboot leaves /mnt/efs an', '# empty local dir and the worker would clone a divergent remote.git', '# onto the instance disk, invisible to the Lambda.', `echo '${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH} efs _netdev,${efsMountOptions} 0 0' >> /etc/fstab`, '', '# The bundle runs only if its sha256 is the one given here (synthesized,', '# or the workerCode parameter); a mismatch fails the boot under the trap', '# above.', `retry 'worker bundle download' aws s3 cp s3://${workerBundle.bucketName}/${workerBundle.objectKey} ${WORKER_BUNDLE_DOWNLOAD_PATH}`, `echo '${workerBundle.sha256}  ${WORKER_BUNDLE_DOWNLOAD_PATH}' | sha256sum -c -`, '# Root-owned, and read-only to the service (ProtectSystem=strict below).', `install -D -m 0644 ${WORKER_BUNDLE_DOWNLOAD_PATH} /opt/canopy-worker/index.js`, '# The worker bundle is ESM (esbuild --format=esm). Without this marker a', '# .js file is CommonJS by default and the import statement fails.', '# Node >=22.7 auto-detects module syntax and would mask its absence, and', '# this instance now runs node-22 -- so the marker is kept BECAUSE that', '# auto-detection is a fallback we should not depend on, not because the', '# runtime here still needs it.', `echo '{"type":"module"}' > /opt/canopy-worker/package.json`, '', '# Write environment file for systemd service', `cat > /opt/canopy-worker/.env << 'ENVEOF'`, envFileContent, 'ENVEOF', '', '# Create systemd service', `cat > /etc/systemd/system/canopy-worker.service << 'SVCEOF'`, '[Unit]', 'Description=CanopyCMS Worker Daemon', 'After=network.target', '# Never run against an unmounted /mnt/efs (see the fstab note above).', `RequiresMountsFor=${EFS_MOUNT_PATH}`, '', '[Service]', 'Type=simple', 'User=ec2-user', 'WorkingDirectory=/opt/canopy-worker', '# Namespaced binary, not bare `node`: AL2023 points /usr/bin/node at', '# an installed version through `alternatives`, and AWS documents that', '# selection as able to change at any time. node-22 always means 22.', 'ExecStart=/usr/bin/node-22 index.js', 'Restart=always', 'RestartSec=5', 'TimeoutStartSec=300', '# Stopping drains: SIGTERM reaches node ONLY (mixed), so the git', '# children it waits for keep running. Whatever is left when node', '# exits, or at TimeoutStopSec, is SIGKILLed; 120s outlasts the 90s', '# drain and its abort grace.', 'KillMode=mixed', 'TimeoutStopSec=120', '# A worker that drained for an instance termination exits with this;', '# restarting it would start work on an instance about to disappear.', `RestartPreventExitStatus=${EXIT_DRAINED_FOR_TERMINATION}`, `SuccessExitStatus=${EXIT_DRAINED_FOR_TERMINATION}`, '# File output, not journal: the CloudWatch agent cannot read journald,', '# so it tails this file instead (see the agent config below).', '# systemd opens append: targets as root, following symlinks, before it', '# applies the sandbox or drops to User=. So the worker must own neither', '# the file nor its directory: one that could swap worker.log for a', '# symlink would have its output appended to any file on the host as root.', '# Hence no LogsDirectory=, which chowns the directory to User=; user-data', '# creates both, root-owned, before the first start. The worker writes', '# through the descriptor systemd hands it.', 'StandardOutput=append:/var/log/canopy-worker/worker.log', 'StandardError=append:/var/log/canopy-worker/worker.log', '# The private GitHub mirror, /var/lib/canopy-worker on the root volume:', '# the only git repository the GitHub credential is used in. The worker', '# does not start without it.', 'StateDirectory=canopy-worker', 'EnvironmentFile=/opt/canopy-worker/.env', '# What this unit provides; a bundle needing more refuses to start.', `Environment=${WORKER_CONTRACT_ENV}=${WORKER_CONTRACT_VERSION}`, '# Sandbox. Writable: the workspace, the state dir (StateDirectory=) and a', '# private /tmp.', '# ProtectHome=tmpfs, not yes: every git call reads per-user files under', '# $HOME (~/.config/git/attributes, ignore), and an inaccessible /home', '# makes each read warn "Permission denied"; an empty one is a silent', '# ENOENT.', ...WORKER_SANDBOX_DIRECTIVES, `ReadWritePaths=${EFS_MOUNT_PATH}`, '', '[Install]', 'WantedBy=multi-user.target', 'SVCEOF', '', '# The worker log, root-owned (see the unit), and created BEFORE the first', '# systemctl start: without the directory the append: open fails with', '# 209/STDOUT and Restart=always crash-loops forever. 0644, so it stays', '# readable without sudo.', 'install -d -o root -g root -m 0755 /var/log/canopy-worker', 'install -o root -g root -m 0644 /dev/null /var/log/canopy-worker/worker.log', '', '# In the worker log format (canopycms worker/log.ts), so the CloudWatch', '# agent ships it as its own event: the agent reads a file it has no saved', '# position for from the start (its `from_beginning` defaults to true).', 'if [ "$WORKER_UNPATCHED" = 1 ]; then', `  echo "$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) ERROR canopy-worker boot: 'dnf upgrade' failed; ${WORKER_UNPATCHED_LOG_PHRASE}" >> /var/log/canopy-worker/worker.log`, 'fi', '', '# Start worker', 'systemctl daemon-reload', 'systemctl enable canopy-worker', 'systemctl start canopy-worker', '', '# ---- CloudWatch log shipping ----', '# Placed AFTER worker start: with set -euo pipefail, a package/agent', '# failure here must not prevent the worker from running (shipping is', '# best-effort).', '#', '# DISARM THE FAIL-FAST TRAP AND errexit FIRST. Everything above this line', '# is required for the worker to exist at all, so a failure there should', '# replace the instance. Nothing below it is: the worker is already', '# running and healthy by this point. Leaving the trap armed would let a', '# package-mirror outage during the agent install shut down a perfectly', '# good worker -- and the ASG would relaunch straight into the same', '# outage, turning degraded log shipping into a replacement loop with the', '# worker down for its duration. That is strictly worse than the', '# best-effort behaviour this section has always documented.', 'trap - ERR', 'set +e', "retry 'dnf install amazon-cloudwatch-agent logrotate' dnf install -y amazon-cloudwatch-agent logrotate", '', '# Bound on-disk growth; copytruncate keeps the fd the CW agent tails valid', '# (tiny copy->truncate loss window is acceptable for diagnostic logs).', `cat > /etc/logrotate.d/canopy-worker << 'ROTEOF'`, '/var/log/canopy-worker/worker.log {', '    size 10M', '    rotate 5', '    compress', '    copytruncate', '    missingok', '    notifempty', '}', 'ROTEOF', '# The config above only fires when logrotate actually runs; AL2023', '# presets may leave logrotate.timer disabled, and without it the size', '# cap never triggers and worker.log grows until the nano disk fills.', '# --now is idempotent if the timer is already enabled/running.', 'systemctl enable --now logrotate.timer', '', '# No retention_in_days here: CDK owns retention on the pre-created group.', `cat > /opt/aws/amazon-cloudwatch-agent/etc/canopy-worker-logs.json << 'CWEOF'`, '{', '  "logs": {', '    "logs_collected": {', '      "files": {', '        "collect_list": [', '          {', '            "file_path": "/var/log/canopy-worker/worker.log",', `            "log_group_name": "${this.workerLogGroup.logGroupName}",`, '            "log_stream_name": "{instance_id}",', 
         // The worker prefixes every line with an ISO-8601 timestamp
         // (packages/canopycms/src/worker/log.ts). Parsing it here is what makes
         // CloudWatch show the time the WORKER emitted a line rather than the

package/dist/constructs/worker-bundle.js

@@ -3,6 +3,7 @@ import { readFileSync } from 'node:fs';
 import * as path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { CfnCondition, CfnOutput, CfnParameter, Duration, Fn, RemovalPolicy, Token, aws_iam as iam, aws_s3 as s3, aws_s3_assets as s3assets, } from 'aws-cdk-lib';
+import { WORKER_CONTRACT_VERSION } from './worker-lifecycle.js';
 // This package is `"type": "module"`, so `__dirname` is not a global in its
 // compiled output. Vitest shims it, so this file's tests would not notice.
 // Same fix as ./asset-support.ts.
@@ -73,6 +74,13 @@ export function workerBundleSource(scope, workerCode, workerRole) {
         description: 'The template parameter that selects the worker bundle',
         value: sha256Parameter.logicalId,
     });
+    // Read by a change-set gate before a parameter-only roll: the bundle's
+    // `index.js.contract` must not exceed it, because such a change set runs the
+    // new bundle under this template's unit.
+    new CfnOutput(scope, 'WorkerContract', {
+        description: 'The worker contract version this template provides',
+        value: String(WORKER_CONTRACT_VERSION),
+    });
     new CfnOutput(scope, 'WorkerBundleBucketName', {
         description: `The bucket holding worker bundles, as ${WORKER_BUNDLE_KEY_PREFIX}<sha256>.js`,
         value: bundleBucket.bucketName,

package/dist/constructs/worker-lifecycle.d.ts

@@ -24,6 +24,26 @@ export declare const EXIT_WORKER_SELF_STOPPED = 69;
  * arms the watch for a spot interruption notice.
  */
 export declare const WORKER_CAPACITY_ENV = "CANOPYCMS_WORKER_CAPACITY";
+/**
+ * The worker contract version: stamped into the unit by the construct (and,
+ * in parameter mode, into the `WorkerContract` stack output), and published beside the
+ * bundle as `worker/dist/index.js.contract`, so a change-set gate can compare
+ * the two before rolling a bundle. See {@link WORKER_CONTRACT_REQUIREMENTS}.
+ */
+export declare const WORKER_CONTRACT_VERSION: 1;
+/** The unit's `Environment=` variable carrying its worker contract version. */
+export declare const WORKER_CONTRACT_ENV = "CANOPYCMS_WORKER_CONTRACT";
+/**
+ * The unit's worker contract version, read from the worker's environment as
+ * {@link WORKER_CONTRACT_REQUIREMENTS} describes, or `NaN` for a stamp that is
+ * not a whole number.
+ */
+export declare function unitWorkerContract(env: Readonly<Record<string, string | undefined>>): number;
+/**
+ * The fatal line for a unit older than this bundle, or `undefined` when the
+ * unit's contract is new enough.
+ */
+export declare function workerContractShortfall(env: Readonly<Record<string, string | undefined>>): string | undefined;
 /**
  * The text of the line `syncGit()` (canopycms `worker/git-sync.ts`) logs at the
  * start of every git-sync cycle. The worker-down alarm's metric filter counts

package/dist/constructs/worker-lifecycle.js

@@ -24,6 +24,62 @@ export const EXIT_WORKER_SELF_STOPPED = 69;
  * arms the watch for a spot interruption notice.
  */
 export const WORKER_CAPACITY_ENV = 'CANOPYCMS_WORKER_CAPACITY';
+/**
+ * What each worker contract version requires of the unit and template, in
+ * order: entry `i` is version `i + 1`. The construct stamps the latest version
+ * into every unit it writes, and a bundle refuses to start under a unit whose
+ * version is lower.
+ *
+ * A unit with no stamp predates the stamp, so its version is read from what
+ * it observably provides: 1 when `STATE_DIRECTORY` is set (systemd sets it
+ * from `StateDirectory=`), else 0. Inference never reads past 1, so from
+ * version 2 on only an explicit stamp satisfies a bundle.
+ *
+ * Append an entry ONLY when the bundle starts to need something new from the
+ * unit or template. Anything a bundle merely tolerates the absence of does
+ * not count. The bundle's "template too old" line names every entry above the
+ * unit's version, so write it as the setting an operator would add.
+ */
+const WORKER_CONTRACT_REQUIREMENTS = ['StateDirectory=canopy-worker'];
+/**
+ * The worker contract version: stamped into the unit by the construct (and,
+ * in parameter mode, into the `WorkerContract` stack output), and published beside the
+ * bundle as `worker/dist/index.js.contract`, so a change-set gate can compare
+ * the two before rolling a bundle. See {@link WORKER_CONTRACT_REQUIREMENTS}.
+ */
+export const WORKER_CONTRACT_VERSION = WORKER_CONTRACT_REQUIREMENTS.length;
+/** The unit's `Environment=` variable carrying its worker contract version. */
+export const WORKER_CONTRACT_ENV = 'CANOPYCMS_WORKER_CONTRACT';
+/**
+ * The unit's worker contract version, read from the worker's environment as
+ * {@link WORKER_CONTRACT_REQUIREMENTS} describes, or `NaN` for a stamp that is
+ * not a whole number.
+ */
+export function unitWorkerContract(env) {
+    const stamp = env[WORKER_CONTRACT_ENV];
+    if (stamp === undefined)
+        return env.STATE_DIRECTORY ? 1 : 0;
+    return /^\d+$/.test(stamp) ? Number(stamp) : NaN;
+}
+/**
+ * The fatal line for a unit older than this bundle, or `undefined` when the
+ * unit's contract is new enough.
+ */
+export function workerContractShortfall(env) {
+    const unit = unitWorkerContract(env);
+    if (Number.isNaN(unit)) {
+        return `canopy-worker: ${WORKER_CONTRACT_ENV}=${JSON.stringify(env[WORKER_CONTRACT_ENV])} on the worker unit is not a whole number`;
+    }
+    if (unit >= WORKER_CONTRACT_VERSION)
+        return undefined;
+    return (`canopy-worker: template too old for this bundle: needs worker contract ` +
+        `${WORKER_CONTRACT_VERSION}, unit has ${unit} (` +
+        WORKER_CONTRACT_REQUIREMENTS.slice(unit)
+            .map((setting, i) => `contract ${unit + i + 1} adds ${setting}`)
+            .join('; ') +
+        `). Deploy the stack template ` +
+        `before rolling this bundle.`);
+}
 /**
  * The text of the line `syncGit()` (canopycms `worker/git-sync.ts`) logs at the
  * start of every git-sync cycle. The worker-down alarm's metric filter counts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "canopycms-cdk",
-  "version": "0.0.68-int.111",
+  "version": "0.0.68-int.113",
   "description": "AWS CDK constructs and EC2 worker for CanopyCMS deployment",
   "license": "MIT",
   "repository": {
@@ -31,7 +31,7 @@
   "peerDependencies": {
     "aws-cdk-lib": "^2.260.0",
     "constructs": "^10.0.0",
-    "canopycms": "0.0.68-int.111"
+    "canopycms": "0.0.68-int.113"
   },
   "peerDependenciesMeta": {
     "aws-cdk-lib": {
@@ -57,15 +57,15 @@
     "tsx": "^4.19.0",
     "typescript": "^5.6.3",
     "vitest": "^4.1.0",
-    "canopycms": "0.0.68-int.111",
-    "canopycms-auth-clerk": "0.0.68-int.111"
+    "canopycms": "0.0.68-int.113",
+    "canopycms-auth-clerk": "0.0.68-int.113"
   },
   "scripts": {
     "build": "rm -rf dist && tsc && node ../../scripts/add-js-extensions.mjs dist",
     "build:worker": "esbuild worker/index.ts --bundle --platform=node --target=node22 --format=esm --outfile=worker/dist/index.js --banner:js=\"import { createRequire } from 'module'; const require = createRequire(import.meta.url);\"",
-    "build:worker-checksum": "node worker/write-bundle-checksum.mjs",
+    "build:worker-metadata": "node --import tsx worker/write-bundle-metadata.ts",
     "build:lambda": "node lambda/asset-transform/build.mjs",
-    "build:test-fixtures": "pnpm run build:worker && pnpm run build:worker-checksum && node lambda/asset-transform/build.mjs --skip-native",
+    "build:test-fixtures": "pnpm run build:worker && pnpm run build:worker-metadata && node lambda/asset-transform/build.mjs --skip-native",
     "lint": "eslint src/ 'lambda/**/*.ts' 'canary/**/*.ts' 'worker/**/*.ts' 'test-support/**/*.ts'",
     "test": "pnpm run build:test-fixtures && vitest run",
     "typecheck": "tsc --noEmit && tsc --noEmit -p lambda/tsconfig.json && tsc --noEmit -p canary/tsconfig.json && tsc --noEmit -p worker/tsconfig.json && tsc --noEmit -p test-support/tsconfig.json"

package/worker/canopy-worker.service

@@ -26,13 +26,15 @@ SuccessExitStatus=75
 # File output, not journal: the CloudWatch agent cannot read journald,
 # so it tails this file instead (agent config lives in the cms-service.ts
 # user-data, which writes the deployed copy of this unit).
-# CAUTION: /var/log/canopy-worker must exist BEFORE first start. systemd
-# opens append: targets before it creates LogsDirectory= dirs
-# (systemd#27591), so without the pre-created dir the exec fails with
-# 209/STDOUT and Restart=always crash-loops forever. The deployed user-data
-# runs mkdir first; do the same if you install this unit by hand.
-# LogsDirectory= is kept for ownership management on subsequent starts.
-LogsDirectory=canopy-worker
+# systemd opens append: targets as root, following symlinks, before it
+# applies the sandbox or drops to User=. So the worker must own neither the
+# file nor its directory: one that could swap worker.log for a symlink would
+# have its output appended to any file on the host as root. Hence no
+# LogsDirectory=, which chowns the directory to User=. Create both, root-owned,
+# BEFORE the first start (without the directory the exec fails with 209/STDOUT
+# and Restart=always crash-loops):
+#   install -d -o root -g root -m 0755 /var/log/canopy-worker
+#   install -o root -g root -m 0644 /dev/null /var/log/canopy-worker/worker.log
 StandardOutput=append:/var/log/canopy-worker/worker.log
 StandardError=append:/var/log/canopy-worker/worker.log
 
@@ -43,9 +45,12 @@ StateDirectory=canopy-worker
 
 # Environment
 EnvironmentFile=/opt/canopy-worker/.env
+# The worker contract this unit provides (WORKER_CONTRACT_VERSION in
+# src/constructs/worker-lifecycle.ts); a bundle needing more refuses to start.
+Environment=CANOPYCMS_WORKER_CONTRACT=1
 
-# Sandbox. Writable: the workspace, the log dir (LogsDirectory=), the state dir
-# (StateDirectory=) and a private /tmp.
+# Sandbox. Writable: the workspace, the state dir (StateDirectory=) and a
+# private /tmp.
 # ProtectHome=tmpfs, not yes: every git call reads per-user files under $HOME
 # (~/.config/git/attributes, ignore), and an inaccessible /home makes each read
 # warn "Permission denied"; an empty one is a silent ENOENT.