canopycms-cdk 0.0.68-int.110 → 0.0.68-int.112

package/dist/constructs/cms-service.d.ts

@@ -362,6 +362,9 @@ export interface CanopyCmsServiceProps {
      * minutes, which covers a crash loop, a boot loop, a worker that never
      * started and a worker whose loops have stopped. A deploy or a spot
      * replacement does not trip it. See `workerDownAlarm`.
+     *
+     * The topic is also notified when a worker boot could not upgrade its
+     * packages and started on the AMI's. See `workerUnpatchedAlarm`.
      */
     alarmTopic?: sns.ITopic;
     /**
@@ -482,6 +485,8 @@ export declare class CanopyCmsService extends Construct {
     readonly workerLogGroup: logs.LogGroup;
     /** Alarms when the worker logs no git-sync cycle for 30 minutes. Set only with `alarmTopic`. */
     readonly workerDownAlarm?: cloudwatch.Alarm;
+    /** Alarms when a worker boot started without its package upgrade. Set only with `alarmTopic`. */
+    readonly workerUnpatchedAlarm?: cloudwatch.Alarm;
     /**
      * With `workerCode: { source: 'parameter' }`: the bucket worker bundles are
      * uploaded to, as `canopy-worker/<sha256>.js`.

package/dist/constructs/cms-service.js

@@ -3,7 +3,7 @@ import { Annotations, ArnFormat, Duration, Names, RemovalPolicy, Stack, Token, a
 import { attachLambdaExecutionPolicies } from './lambda-execution-role.js';
 import { WORKER_BUNDLE_DOWNLOAD_PATH, workerBundleSource } from './worker-bundle.js';
 import { attachEditorBehaviors } from './editor-routing.js';
-import { EXIT_DRAINED_FOR_TERMINATION, WORKER_CAPACITY_ENV, WORKER_DRAIN_HOOK_NAME, WORKER_SYNC_LOG_PHRASE, } from './worker-lifecycle.js';
+import { EXIT_DRAINED_FOR_TERMINATION, WORKER_CAPACITY_ENV, WORKER_CONTRACT_ENV, WORKER_CONTRACT_VERSION, WORKER_DRAIN_HOOK_NAME, WORKER_SYNC_LOG_PHRASE, } from './worker-lifecycle.js';
 /**
  * Synth-time mirror of `resolveDeploymentName`'s rule in the `canopycms`
  * package (packages/canopycms/src/operating-mode/deployment-name.ts).
@@ -37,6 +37,12 @@ const isValidDeploymentName = (name) => /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(name
  * by one process must resolve for the other.
  */
 const EFS_MOUNT_PATH = '/mnt/efs';
+/**
+ * The text of the worker.log line user data writes when the boot's package
+ * upgrade failed and the worker runs on the AMI's packages. The unpatched
+ * alarm's metric filter counts it.
+ */
+const WORKER_UNPATCHED_LOG_PHRASE = 'running unpatched on the AMI packages until the next boot';
 /**
  * The worker unit's sandbox, also in worker/canopy-worker.service. Node and git
  * need none of what these take away; `MemoryDenyWriteExecute` is left out
@@ -989,6 +995,28 @@ export class CanopyCmsService extends Construct {
             const topicAction = new cloudwatchActions.SnsAction(props.alarmTopic);
             this.workerDownAlarm.addAlarmAction(topicAction);
             this.workerDownAlarm.addOkAction(topicAction);
+            const unpatchedPrefix = 'WorkerUnpatchedBoots-';
+            const unpatchedBoots = new logs.MetricFilter(this, 'WorkerUnpatchedBoots', {
+                logGroup: this.workerLogGroup,
+                filterPattern: logs.FilterPattern.literal(`"${WORKER_UNPATCHED_LOG_PHRASE}"`),
+                metricNamespace: 'CanopyCMS',
+                metricName: `${unpatchedPrefix}${Names.uniqueResourceName(this, { maxLength: 255 - unpatchedPrefix.length })}`,
+                metricValue: '1',
+            });
+            // One line per unpatched boot, so the alarm returns to OK one period
+            // later on its own; an OK notification would only repeat the alarm.
+            this.workerUnpatchedAlarm = new cloudwatch.Alarm(this, 'WorkerUnpatchedAlarm', {
+                metric: unpatchedBoots.metric({ statistic: 'Sum', period: Duration.minutes(10) }),
+                threshold: 1,
+                comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
+                evaluationPeriods: 1,
+                treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
+                alarmDescription: `A CMS worker boot could not upgrade its packages and started on the AMI's. It is ` +
+                    `working, and the next boot retries the upgrade. The boot's console output ` +
+                    `(aws ec2 get-console-output) and the worker log group ` +
+                    `${this.workerLogGroup.logGroupName} name the failure.`,
+            });
+            this.workerUnpatchedAlarm.addAlarmAction(topicAction);
         }
         const workerBundle = workerBundleSource(this, props.workerCode ?? { source: 'asset' }, workerRole);
         this.workerBundleBucket = workerBundle.bundleBucket;
@@ -1069,7 +1097,7 @@ export class CanopyCmsService extends Construct {
             .join('\n');
         const efsMountOptions = `tls,iam,accesspoint=${accessPoint.accessPointId}`;
         const userData = ec2.UserData.forLinux();
-        userData.addCommands('#!/bin/bash', 'set -euo pipefail', '', '# FAIL-FAST. Every step below is required for the worker to exist at', '# all, and until this trap existed a failure in any of them aborted', '# user-data BEFORE the systemd unit was written -- leaving an instance', "# that runs, passes the ASG's EC2-only health check indefinitely, and", '# does nothing. cfn-signal is deliberately not used here (it would', '# prove nothing about READINESS, which is the argument recorded below),', '# but that argument never covered a boot-SCRIPT failure: `cdk deploy`', '# reported success while publishes queued on EFS, the auth cache went', '# stale and PRs stopped being created, until a human noticed the admin', '# panel showing the worker absent.', '#', '# Shutting down makes the instance fail its EC2 health check, so the ASG', '# replaces it -- which is the only automatic recovery available in this', '# topology. The echo lands in the console log, readable via', '# `aws ec2 get-console-output`, because the CloudWatch agent is itself', '# configured further down and cannot be relied on to exist yet.', 'trap \'echo "canopy-worker user-data FAILED at line $LINENO (exit $?)" >&2; shutdown -h now\' ERR', '', '# Bounded retry for the network-dependent steps. Package mirrors and S3', '# have transient failures; a single flake should cost seconds, not an', '# instance replacement.', 'retry() {', '  local n=0', '  until "$@"; do', '    n=$((n + 1))', '    if [ "$n" -ge 5 ]; then', '      echo "canopy-worker: command failed after $n attempts: $*" >&2', '      return 1', '    fi', '    sleep $((n * 5))', '  done', '}', '', '# Patch first. AL2023 locks dnf to the repository release its AMI was', '# built from, so a relaunch from the AMI resolved at the last deploy', '# would otherwise install nothing newer than that AMI. Moving to the', '# latest release also makes it the one every install below resolves', '# from. The kernel is left out: a new one applies only at a reboot,', '# which nothing here performs, so it would cost boot time and change', '# nothing. Kernel fixes arrive with the AMI a deploy resolves.', "retry dnf upgrade --releasever=latest --exclude='kernel*' -y", '', 'retry dnf install -y git', "# Node comes from AL2023's own repos, NOT a piped third-party installer.", '# The previous boot curled the NodeSource RPM setup script straight into', '# bash, so every instance replacement -- which the ASG performs on every', '# `cdk deploy`, plus every spot interruption -- depended on a third party', '# being reachable. cms-deploy.test.ts asserts that URL never comes back,', '# which is why it is not spelled out here.', '#', '# nodejs22, not 20: Node 20 reached upstream EOL on 2026-04-30, and this', '# repo declares `engines.node: ">=22"` with .nvmrc `v22`, so the worker', '# was running an EOL runtime BELOW the floor its own code is tested at.', "# 22 (EOL 2027-04-30) matches .nvmrc, CI, and the transform Lambda's", '# NODEJS_22_X, so one runtime is tested everywhere.', '#', '# ExecStart uses the NAMESPACED /usr/bin/node-22 (see the systemd unit', '# below), not the bare `node`: AL2023 installs versioned binaries and', '# points `/usr/bin/node` at one of them through `alternatives`, whose', '# selection AWS documents as able to change at any time.', 'retry dnf install -y nodejs22', '', '# Mount EFS through the same access point, at the same path, as the', '# Lambda (see EFS_MOUNT_PATH). amazon-efs-utils refuses an access-point', '# mount without tls (efs_utils_common/mount_options.py).', 'retry dnf install -y amazon-efs-utils', `mkdir -p ${EFS_MOUNT_PATH}`, '# Retried: an `iam` mount also fetches instance-role credentials.', `retry mount -t efs -o ${efsMountOptions} ${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH}`, '# Persist the mount across instance reboots: user-data runs once per', '# instance, so without an fstab entry a plain reboot leaves /mnt/efs an', '# empty local dir and the worker would clone a divergent remote.git', '# onto the instance disk, invisible to the Lambda.', `echo '${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH} efs _netdev,${efsMountOptions} 0 0' >> /etc/fstab`, '', '# The bundle runs only if its sha256 is the one given here (synthesized,', '# or the workerCode parameter); a mismatch fails the boot under the trap', '# above.', `retry aws s3 cp s3://${workerBundle.bucketName}/${workerBundle.objectKey} ${WORKER_BUNDLE_DOWNLOAD_PATH}`, `echo '${workerBundle.sha256}  ${WORKER_BUNDLE_DOWNLOAD_PATH}' | sha256sum -c -`, '# Root-owned, and read-only to the service (ProtectSystem=strict below).', `install -D -m 0644 ${WORKER_BUNDLE_DOWNLOAD_PATH} /opt/canopy-worker/index.js`, '# The worker bundle is ESM (esbuild --format=esm). Without this marker a', '# .js file is CommonJS by default and the import statement fails.', '# Node >=22.7 auto-detects module syntax and would mask its absence, and', '# this instance now runs node-22 -- so the marker is kept BECAUSE that', '# auto-detection is a fallback we should not depend on, not because the', '# runtime here still needs it.', `echo '{"type":"module"}' > /opt/canopy-worker/package.json`, '', '# Write environment file for systemd service', `cat > /opt/canopy-worker/.env << 'ENVEOF'`, envFileContent, 'ENVEOF', '', '# Create systemd service', `cat > /etc/systemd/system/canopy-worker.service << 'SVCEOF'`, '[Unit]', 'Description=CanopyCMS Worker Daemon', 'After=network.target', '# Never run against an unmounted /mnt/efs (see the fstab note above).', `RequiresMountsFor=${EFS_MOUNT_PATH}`, '', '[Service]', 'Type=simple', 'User=ec2-user', 'WorkingDirectory=/opt/canopy-worker', '# Namespaced binary, not bare `node`: AL2023 points /usr/bin/node at', '# an installed version through `alternatives`, and AWS documents that', '# selection as able to change at any time. node-22 always means 22.', 'ExecStart=/usr/bin/node-22 index.js', 'Restart=always', 'RestartSec=5', 'TimeoutStartSec=300', '# Stopping drains: SIGTERM reaches node ONLY (mixed), so the git', '# children it waits for keep running. Whatever is left when node', '# exits, or at TimeoutStopSec, is SIGKILLed; 120s outlasts the 90s', '# drain and its abort grace.', 'KillMode=mixed', 'TimeoutStopSec=120', '# A worker that drained for an instance termination exits with this;', '# restarting it would start work on an instance about to disappear.', `RestartPreventExitStatus=${EXIT_DRAINED_FOR_TERMINATION}`, `SuccessExitStatus=${EXIT_DRAINED_FOR_TERMINATION}`, '# File output, not journal: the CloudWatch agent cannot read journald,', '# so it tails this file instead (see the agent config below).', '# CAUTION: /var/log/canopy-worker must exist BEFORE first start. systemd', '# opens append: targets before it creates LogsDirectory= dirs', '# (systemd#27591), so without the pre-created dir the exec fails with', '# 209/STDOUT and Restart=always crash-loops forever. User-data runs', '# mkdir before systemctl start; LogsDirectory= is kept for ownership.', 'LogsDirectory=canopy-worker', 'StandardOutput=append:/var/log/canopy-worker/worker.log', 'StandardError=append:/var/log/canopy-worker/worker.log', '# The private GitHub mirror, /var/lib/canopy-worker on the root volume:', '# the only git repository the GitHub credential is used in. The worker', '# does not start without it.', 'StateDirectory=canopy-worker', 'EnvironmentFile=/opt/canopy-worker/.env', '# Sandbox. Writable: the workspace, the log dir (LogsDirectory=), the', '# state dir (StateDirectory=) and a private /tmp.', '# ProtectHome=tmpfs, not yes: every git call reads per-user files under', '# $HOME (~/.config/git/attributes, ignore), and an inaccessible /home', '# makes each read warn "Permission denied"; an empty one is a silent', '# ENOENT.', ...WORKER_SANDBOX_DIRECTIVES, `ReadWritePaths=${EFS_MOUNT_PATH}`, '', '[Install]', 'WantedBy=multi-user.target', 'SVCEOF', '', '# Pre-create the worker log dir (crash-loop guard, MUST precede the', '# first systemctl start): systemd opens StandardOutput=append: files', '# BEFORE it creates LogsDirectory= dirs (systemd#27591), so on a fresh', '# instance the unit would fail exec with 209/STDOUT and crash-loop', '# forever without this. The unit keeps LogsDirectory= for ownership', '# management on subsequent starts.', 'mkdir -p /var/log/canopy-worker', 'chown ec2-user:ec2-user /var/log/canopy-worker', '', '# Start worker', 'systemctl daemon-reload', 'systemctl enable canopy-worker', 'systemctl start canopy-worker', '', '# ---- CloudWatch log shipping ----', '# Placed AFTER worker start: with set -euo pipefail, a package/agent', '# failure here must not prevent the worker from running (shipping is', '# best-effort).', '#', '# DISARM THE FAIL-FAST TRAP AND errexit FIRST. Everything above this line', '# is required for the worker to exist at all, so a failure there should', '# replace the instance. Nothing below it is: the worker is already', '# running and healthy by this point. Leaving the trap armed would let a', '# package-mirror outage during the agent install shut down a perfectly', '# good worker -- and the ASG would relaunch straight into the same', '# outage, turning degraded log shipping into a replacement loop with the', '# worker down for its duration. That is strictly worse than the', '# best-effort behaviour this section has always documented.', 'trap - ERR', 'set +e', 'retry dnf install -y amazon-cloudwatch-agent logrotate', '', '# Bound on-disk growth; copytruncate keeps the fd the CW agent tails valid', '# (tiny copy->truncate loss window is acceptable for diagnostic logs).', `cat > /etc/logrotate.d/canopy-worker << 'ROTEOF'`, '/var/log/canopy-worker/worker.log {', '    size 10M', '    rotate 5', '    compress', '    copytruncate', '    missingok', '    notifempty', '}', 'ROTEOF', '# The config above only fires when logrotate actually runs; AL2023', '# presets may leave logrotate.timer disabled, and without it the size', '# cap never triggers and worker.log grows until the nano disk fills.', '# --now is idempotent if the timer is already enabled/running.', 'systemctl enable --now logrotate.timer', '', '# No retention_in_days here: CDK owns retention on the pre-created group.', `cat > /opt/aws/amazon-cloudwatch-agent/etc/canopy-worker-logs.json << 'CWEOF'`, '{', '  "logs": {', '    "logs_collected": {', '      "files": {', '        "collect_list": [', '          {', '            "file_path": "/var/log/canopy-worker/worker.log",', `            "log_group_name": "${this.workerLogGroup.logGroupName}",`, '            "log_stream_name": "{instance_id}",', 
+        userData.addCommands('#!/bin/bash', 'set -euo pipefail', '', '# FAIL-FAST. Every step below is required for the worker to exist at', '# all, except the swap and the package upgrade, which are written as', '# conditions so that they cannot trip it. Until this trap existed, a', '# failure in any required step aborted user-data BEFORE the systemd unit was written -- leaving an instance', "# that runs, passes the ASG's EC2-only health check indefinitely, and", '# does nothing. cfn-signal is deliberately not used here (it would', '# prove nothing about READINESS, which is the argument recorded below),', '# but that argument never covered a boot-SCRIPT failure: `cdk deploy`', '# reported success while publishes queued on EFS, the auth cache went', '# stale and PRs stopped being created, until a human noticed the admin', '# panel showing the worker absent.', '#', '# Shutting down makes the instance fail its EC2 health check, so the ASG', '# replaces it -- which is the only automatic recovery available in this', '# topology. The echo lands in the console log, readable via', '# `aws ec2 get-console-output`, because the CloudWatch agent is itself', '# configured further down and cannot be relied on to exist yet.', 'trap \'echo "canopy-worker user-data FAILED at line $LINENO (exit $?)" >&2; shutdown -h now\' ERR', '', '# Bounded retry for the network-dependent steps. Package mirrors and S3', '# have transient failures; a single flake should cost seconds, not an', '# instance replacement. The first argument names the step in the message', '# an exhausted step leaves on the console.', 'retry() {', '  local step=$1 n=0', '  shift', '  until "$@"; do', '    n=$((n + 1))', '    if [ "$n" -ge 5 ]; then', `      echo "canopy-worker boot: '$step' failed after $n attempts" >&2`, '      return 1', '    fi', '    sleep $((n * 5))', '  done', '}', '', '# Swap, before the first dnf: loading the repository metadata takes more', '# memory than a t4g.nano has free after boot, and the kernel OOM-kills', '# dnf. It stays on afterwards at low swappiness, as headroom for git and', '# the CloudWatch agent rather than working memory. It is on the encrypted', '# root volume, so memory paged out of the worker is encrypted at rest.', '# fallocate is safe here: XFS, the root file system, supports', '# preallocated swap files. A failure only warns, because dnf may still fit', '# and a boot that stopped here would put the group into a replacement', '# loop. Each command carries `|| return 1` because errexit does not apply', '# inside a function called as a condition.', 'setup_swap() {', '  swapon --show=NAME --noheadings | grep -qx /swapfile && return 0', '  if [ "$(stat -c %s /swapfile 2>/dev/null)" != 1073741824 ]; then', '    # The file, plus 2 GiB for packages, logs and the GitHub mirror.', '    local avail_mib', `    avail_mib=$(df --output=avail -m / | tail -n 1 | tr -d ' ')`, '    case "$avail_mib" in ""|*[!0-9]*) return 1 ;; esac', '    if [ "$avail_mib" -lt 3072 ]; then', '      echo "canopy-worker boot: under 3 GiB free on /, so no swap file" >&2', '      return 1', '    fi', '    rm -f /swapfile || return 1', '    fallocate -l 1G /swapfile || return 1', '  fi', '  chmod 600 /swapfile || return 1', '  mkswap /swapfile || return 1', '  # swapon last, so that swap is never on when this returns 1.', "  grep -qs '^/swapfile ' /etc/fstab || echo '/swapfile none swap defaults 0 0' >> /etc/fstab || return 1", '  mkdir -p /etc/sysctl.d || return 1', "  echo 'vm.swappiness = 10' > /etc/sysctl.d/90-canopy-worker-swap.conf || return 1", '  sysctl -q -w vm.swappiness=10 || return 1', '  swapon /swapfile', '}', 'if ! setup_swap; then', `  echo "canopy-worker boot: 'swap' setup failed; continuing without swap" >&2`, 'fi', '', '# Patch first. AL2023 locks dnf to the repository release its AMI was', '# built from, so a relaunch from the AMI resolved at the last deploy', '# would otherwise install nothing newer than that AMI. Moving to the', '# latest release also makes it the one every install below resolves', '# from. The kernel is left out: a new one applies only at a reboot,', '# which nothing here performs, so it would cost boot time and change', '# nothing. Kernel fixes arrive with the AMI a deploy resolves.', '#', "# A failed upgrade does not stop the boot: the worker starts on the AMI's", '# packages and the next boot (by default the weekly recycle at the', '# latest) tries again. Failing here would put the group into a', '# replacement loop with no worker at all. The installs below then', '# resolve from the AMI release: 2023.7.20250428 carries all of them,', '# 2023.7.20250331 lacks nodejs22. The failure is written to worker.log', '# below, for the unpatched alarm.', 'WORKER_UNPATCHED=0', "if ! retry 'dnf upgrade' dnf upgrade --releasever=latest --exclude='kernel*' -y; then", '  WORKER_UNPATCHED=1', `  echo "canopy-worker boot: ${WORKER_UNPATCHED_LOG_PHRASE}" >&2`, 'fi', '', "retry 'dnf install git' dnf install -y git", "# Node comes from AL2023's own repos, NOT a piped third-party installer.", '# The previous boot curled the NodeSource RPM setup script straight into', '# bash, so every instance replacement -- which the ASG performs on every', '# `cdk deploy`, plus every spot interruption -- depended on a third party', '# being reachable. cms-deploy.test.ts asserts that URL never comes back,', '# which is why it is not spelled out here.', '#', '# nodejs22, not 20: Node 20 reached upstream EOL on 2026-04-30, and this', '# repo declares `engines.node: ">=22"` with .nvmrc `v22`, so the worker', '# was running an EOL runtime BELOW the floor its own code is tested at.', "# 22 (EOL 2027-04-30) matches .nvmrc, CI, and the transform Lambda's", '# NODEJS_22_X, so one runtime is tested everywhere.', '#', '# ExecStart uses the NAMESPACED /usr/bin/node-22 (see the systemd unit', '# below), not the bare `node`: AL2023 installs versioned binaries and', '# points `/usr/bin/node` at one of them through `alternatives`, whose', '# selection AWS documents as able to change at any time.', "retry 'dnf install nodejs22' dnf install -y nodejs22", '', '# Mount EFS through the same access point, at the same path, as the', '# Lambda (see EFS_MOUNT_PATH). amazon-efs-utils refuses an access-point', '# mount without tls (efs_utils_common/mount_options.py).', "retry 'dnf install amazon-efs-utils' dnf install -y amazon-efs-utils", `mkdir -p ${EFS_MOUNT_PATH}`, '# Retried: an `iam` mount also fetches instance-role credentials.', `retry 'EFS mount' mount -t efs -o ${efsMountOptions} ${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH}`, '# Persist the mount across instance reboots: user-data runs once per', '# instance, so without an fstab entry a plain reboot leaves /mnt/efs an', '# empty local dir and the worker would clone a divergent remote.git', '# onto the instance disk, invisible to the Lambda.', `echo '${this.fileSystem.fileSystemId}:/ ${EFS_MOUNT_PATH} efs _netdev,${efsMountOptions} 0 0' >> /etc/fstab`, '', '# The bundle runs only if its sha256 is the one given here (synthesized,', '# or the workerCode parameter); a mismatch fails the boot under the trap', '# above.', `retry 'worker bundle download' aws s3 cp s3://${workerBundle.bucketName}/${workerBundle.objectKey} ${WORKER_BUNDLE_DOWNLOAD_PATH}`, `echo '${workerBundle.sha256}  ${WORKER_BUNDLE_DOWNLOAD_PATH}' | sha256sum -c -`, '# Root-owned, and read-only to the service (ProtectSystem=strict below).', `install -D -m 0644 ${WORKER_BUNDLE_DOWNLOAD_PATH} /opt/canopy-worker/index.js`, '# The worker bundle is ESM (esbuild --format=esm). Without this marker a', '# .js file is CommonJS by default and the import statement fails.', '# Node >=22.7 auto-detects module syntax and would mask its absence, and', '# this instance now runs node-22 -- so the marker is kept BECAUSE that', '# auto-detection is a fallback we should not depend on, not because the', '# runtime here still needs it.', `echo '{"type":"module"}' > /opt/canopy-worker/package.json`, '', '# Write environment file for systemd service', `cat > /opt/canopy-worker/.env << 'ENVEOF'`, envFileContent, 'ENVEOF', '', '# Create systemd service', `cat > /etc/systemd/system/canopy-worker.service << 'SVCEOF'`, '[Unit]', 'Description=CanopyCMS Worker Daemon', 'After=network.target', '# Never run against an unmounted /mnt/efs (see the fstab note above).', `RequiresMountsFor=${EFS_MOUNT_PATH}`, '', '[Service]', 'Type=simple', 'User=ec2-user', 'WorkingDirectory=/opt/canopy-worker', '# Namespaced binary, not bare `node`: AL2023 points /usr/bin/node at', '# an installed version through `alternatives`, and AWS documents that', '# selection as able to change at any time. node-22 always means 22.', 'ExecStart=/usr/bin/node-22 index.js', 'Restart=always', 'RestartSec=5', 'TimeoutStartSec=300', '# Stopping drains: SIGTERM reaches node ONLY (mixed), so the git', '# children it waits for keep running. Whatever is left when node', '# exits, or at TimeoutStopSec, is SIGKILLed; 120s outlasts the 90s', '# drain and its abort grace.', 'KillMode=mixed', 'TimeoutStopSec=120', '# A worker that drained for an instance termination exits with this;', '# restarting it would start work on an instance about to disappear.', `RestartPreventExitStatus=${EXIT_DRAINED_FOR_TERMINATION}`, `SuccessExitStatus=${EXIT_DRAINED_FOR_TERMINATION}`, '# File output, not journal: the CloudWatch agent cannot read journald,', '# so it tails this file instead (see the agent config below).', '# systemd opens append: targets as root, following symlinks, before it', '# applies the sandbox or drops to User=. So the worker must own neither', '# the file nor its directory: one that could swap worker.log for a', '# symlink would have its output appended to any file on the host as root.', '# Hence no LogsDirectory=, which chowns the directory to User=; user-data', '# creates both, root-owned, before the first start. The worker writes', '# through the descriptor systemd hands it.', 'StandardOutput=append:/var/log/canopy-worker/worker.log', 'StandardError=append:/var/log/canopy-worker/worker.log', '# The private GitHub mirror, /var/lib/canopy-worker on the root volume:', '# the only git repository the GitHub credential is used in. The worker', '# does not start without it.', 'StateDirectory=canopy-worker', 'EnvironmentFile=/opt/canopy-worker/.env', '# What this unit provides; a bundle needing more refuses to start.', `Environment=${WORKER_CONTRACT_ENV}=${WORKER_CONTRACT_VERSION}`, '# Sandbox. Writable: the workspace, the state dir (StateDirectory=) and a', '# private /tmp.', '# ProtectHome=tmpfs, not yes: every git call reads per-user files under', '# $HOME (~/.config/git/attributes, ignore), and an inaccessible /home', '# makes each read warn "Permission denied"; an empty one is a silent', '# ENOENT.', ...WORKER_SANDBOX_DIRECTIVES, `ReadWritePaths=${EFS_MOUNT_PATH}`, '', '[Install]', 'WantedBy=multi-user.target', 'SVCEOF', '', '# The worker log, root-owned (see the unit), and created BEFORE the first', '# systemctl start: without the directory the append: open fails with', '# 209/STDOUT and Restart=always crash-loops forever. 0644, so it stays', '# readable without sudo.', 'install -d -o root -g root -m 0755 /var/log/canopy-worker', 'install -o root -g root -m 0644 /dev/null /var/log/canopy-worker/worker.log', '', '# In the worker log format (canopycms worker/log.ts), so the CloudWatch', '# agent ships it as its own event: the agent reads a file it has no saved', '# position for from the start (its `from_beginning` defaults to true).', 'if [ "$WORKER_UNPATCHED" = 1 ]; then', `  echo "$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) ERROR canopy-worker boot: 'dnf upgrade' failed; ${WORKER_UNPATCHED_LOG_PHRASE}" >> /var/log/canopy-worker/worker.log`, 'fi', '', '# Start worker', 'systemctl daemon-reload', 'systemctl enable canopy-worker', 'systemctl start canopy-worker', '', '# ---- CloudWatch log shipping ----', '# Placed AFTER worker start: with set -euo pipefail, a package/agent', '# failure here must not prevent the worker from running (shipping is', '# best-effort).', '#', '# DISARM THE FAIL-FAST TRAP AND errexit FIRST. Everything above this line', '# is required for the worker to exist at all, so a failure there should', '# replace the instance. Nothing below it is: the worker is already', '# running and healthy by this point. Leaving the trap armed would let a', '# package-mirror outage during the agent install shut down a perfectly', '# good worker -- and the ASG would relaunch straight into the same', '# outage, turning degraded log shipping into a replacement loop with the', '# worker down for its duration. That is strictly worse than the', '# best-effort behaviour this section has always documented.', 'trap - ERR', 'set +e', "retry 'dnf install amazon-cloudwatch-agent logrotate' dnf install -y amazon-cloudwatch-agent logrotate", '', '# Bound on-disk growth; copytruncate keeps the fd the CW agent tails valid', '# (tiny copy->truncate loss window is acceptable for diagnostic logs).', `cat > /etc/logrotate.d/canopy-worker << 'ROTEOF'`, '/var/log/canopy-worker/worker.log {', '    size 10M', '    rotate 5', '    compress', '    copytruncate', '    missingok', '    notifempty', '}', 'ROTEOF', '# The config above only fires when logrotate actually runs; AL2023', '# presets may leave logrotate.timer disabled, and without it the size', '# cap never triggers and worker.log grows until the nano disk fills.', '# --now is idempotent if the timer is already enabled/running.', 'systemctl enable --now logrotate.timer', '', '# No retention_in_days here: CDK owns retention on the pre-created group.', `cat > /opt/aws/amazon-cloudwatch-agent/etc/canopy-worker-logs.json << 'CWEOF'`, '{', '  "logs": {', '    "logs_collected": {', '      "files": {', '        "collect_list": [', '          {', '            "file_path": "/var/log/canopy-worker/worker.log",', `            "log_group_name": "${this.workerLogGroup.logGroupName}",`, '            "log_stream_name": "{instance_id}",', 
         // The worker prefixes every line with an ISO-8601 timestamp
         // (packages/canopycms/src/worker/log.ts). Parsing it here is what makes
         // CloudWatch show the time the WORKER emitted a line rather than the

package/dist/constructs/worker-bundle.js

@@ -3,6 +3,7 @@ import { readFileSync } from 'node:fs';
 import * as path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { CfnCondition, CfnOutput, CfnParameter, Duration, Fn, RemovalPolicy, Token, aws_iam as iam, aws_s3 as s3, aws_s3_assets as s3assets, } from 'aws-cdk-lib';
+import { WORKER_CONTRACT_VERSION } from './worker-lifecycle.js';
 // This package is `"type": "module"`, so `__dirname` is not a global in its
 // compiled output. Vitest shims it, so this file's tests would not notice.
 // Same fix as ./asset-support.ts.
@@ -73,6 +74,13 @@ export function workerBundleSource(scope, workerCode, workerRole) {
         description: 'The template parameter that selects the worker bundle',
         value: sha256Parameter.logicalId,
     });
+    // Read by a change-set gate before a parameter-only roll: the bundle's
+    // `index.js.contract` must not exceed it, because such a change set runs the
+    // new bundle under this template's unit.
+    new CfnOutput(scope, 'WorkerContract', {
+        description: 'The worker contract version this template provides',
+        value: String(WORKER_CONTRACT_VERSION),
+    });
     new CfnOutput(scope, 'WorkerBundleBucketName', {
         description: `The bucket holding worker bundles, as ${WORKER_BUNDLE_KEY_PREFIX}<sha256>.js`,
         value: bundleBucket.bucketName,

package/dist/constructs/worker-lifecycle.d.ts

@@ -24,6 +24,26 @@ export declare const EXIT_WORKER_SELF_STOPPED = 69;
  * arms the watch for a spot interruption notice.
  */
 export declare const WORKER_CAPACITY_ENV = "CANOPYCMS_WORKER_CAPACITY";
+/**
+ * The worker contract version: stamped into the unit by the construct (and,
+ * in parameter mode, into the `WorkerContract` stack output), and published beside the
+ * bundle as `worker/dist/index.js.contract`, so a change-set gate can compare
+ * the two before rolling a bundle. See {@link WORKER_CONTRACT_REQUIREMENTS}.
+ */
+export declare const WORKER_CONTRACT_VERSION: 1;
+/** The unit's `Environment=` variable carrying its worker contract version. */
+export declare const WORKER_CONTRACT_ENV = "CANOPYCMS_WORKER_CONTRACT";
+/**
+ * The unit's worker contract version, read from the worker's environment as
+ * {@link WORKER_CONTRACT_REQUIREMENTS} describes, or `NaN` for a stamp that is
+ * not a whole number.
+ */
+export declare function unitWorkerContract(env: Readonly<Record<string, string | undefined>>): number;
+/**
+ * The fatal line for a unit older than this bundle, or `undefined` when the
+ * unit's contract is new enough.
+ */
+export declare function workerContractShortfall(env: Readonly<Record<string, string | undefined>>): string | undefined;
 /**
  * The text of the line `syncGit()` (canopycms `worker/git-sync.ts`) logs at the
  * start of every git-sync cycle. The worker-down alarm's metric filter counts

package/dist/constructs/worker-lifecycle.js

@@ -24,6 +24,62 @@ export const EXIT_WORKER_SELF_STOPPED = 69;
  * arms the watch for a spot interruption notice.
  */
 export const WORKER_CAPACITY_ENV = 'CANOPYCMS_WORKER_CAPACITY';
+/**
+ * What each worker contract version requires of the unit and template, in
+ * order: entry `i` is version `i + 1`. The construct stamps the latest version
+ * into every unit it writes, and a bundle refuses to start under a unit whose
+ * version is lower.
+ *
+ * A unit with no stamp predates the stamp, so its version is read from what
+ * it observably provides: 1 when `STATE_DIRECTORY` is set (systemd sets it
+ * from `StateDirectory=`), else 0. Inference never reads past 1, so from
+ * version 2 on only an explicit stamp satisfies a bundle.
+ *
+ * Append an entry ONLY when the bundle starts to need something new from the
+ * unit or template. Anything a bundle merely tolerates the absence of does
+ * not count. The bundle's "template too old" line names every entry above the
+ * unit's version, so write it as the setting an operator would add.
+ */
+const WORKER_CONTRACT_REQUIREMENTS = ['StateDirectory=canopy-worker'];
+/**
+ * The worker contract version: stamped into the unit by the construct (and,
+ * in parameter mode, into the `WorkerContract` stack output), and published beside the
+ * bundle as `worker/dist/index.js.contract`, so a change-set gate can compare
+ * the two before rolling a bundle. See {@link WORKER_CONTRACT_REQUIREMENTS}.
+ */
+export const WORKER_CONTRACT_VERSION = WORKER_CONTRACT_REQUIREMENTS.length;
+/** The unit's `Environment=` variable carrying its worker contract version. */
+export const WORKER_CONTRACT_ENV = 'CANOPYCMS_WORKER_CONTRACT';
+/**
+ * The unit's worker contract version, read from the worker's environment as
+ * {@link WORKER_CONTRACT_REQUIREMENTS} describes, or `NaN` for a stamp that is
+ * not a whole number.
+ */
+export function unitWorkerContract(env) {
+    const stamp = env[WORKER_CONTRACT_ENV];
+    if (stamp === undefined)
+        return env.STATE_DIRECTORY ? 1 : 0;
+    return /^\d+$/.test(stamp) ? Number(stamp) : NaN;
+}
+/**
+ * The fatal line for a unit older than this bundle, or `undefined` when the
+ * unit's contract is new enough.
+ */
+export function workerContractShortfall(env) {
+    const unit = unitWorkerContract(env);
+    if (Number.isNaN(unit)) {
+        return `canopy-worker: ${WORKER_CONTRACT_ENV}=${JSON.stringify(env[WORKER_CONTRACT_ENV])} on the worker unit is not a whole number`;
+    }
+    if (unit >= WORKER_CONTRACT_VERSION)
+        return undefined;
+    return (`canopy-worker: template too old for this bundle: needs worker contract ` +
+        `${WORKER_CONTRACT_VERSION}, unit has ${unit} (` +
+        WORKER_CONTRACT_REQUIREMENTS.slice(unit)
+            .map((setting, i) => `contract ${unit + i + 1} adds ${setting}`)
+            .join('; ') +
+        `). Deploy the stack template ` +
+        `before rolling this bundle.`);
+}
 /**
  * The text of the line `syncGit()` (canopycms `worker/git-sync.ts`) logs at the
  * start of every git-sync cycle. The worker-down alarm's metric filter counts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "canopycms-cdk",
-  "version": "0.0.68-int.110",
+  "version": "0.0.68-int.112",
   "description": "AWS CDK constructs and EC2 worker for CanopyCMS deployment",
   "license": "MIT",
   "repository": {
@@ -31,7 +31,7 @@
   "peerDependencies": {
     "aws-cdk-lib": "^2.260.0",
     "constructs": "^10.0.0",
-    "canopycms": "0.0.68-int.110"
+    "canopycms": "0.0.68-int.112"
   },
   "peerDependenciesMeta": {
     "aws-cdk-lib": {
@@ -57,15 +57,15 @@
     "tsx": "^4.19.0",
     "typescript": "^5.6.3",
     "vitest": "^4.1.0",
-    "canopycms": "0.0.68-int.110",
-    "canopycms-auth-clerk": "0.0.68-int.110"
+    "canopycms": "0.0.68-int.112",
+    "canopycms-auth-clerk": "0.0.68-int.112"
   },
   "scripts": {
     "build": "rm -rf dist && tsc && node ../../scripts/add-js-extensions.mjs dist",
     "build:worker": "esbuild worker/index.ts --bundle --platform=node --target=node22 --format=esm --outfile=worker/dist/index.js --banner:js=\"import { createRequire } from 'module'; const require = createRequire(import.meta.url);\"",
-    "build:worker-checksum": "node worker/write-bundle-checksum.mjs",
+    "build:worker-metadata": "node --import tsx worker/write-bundle-metadata.ts",
     "build:lambda": "node lambda/asset-transform/build.mjs",
-    "build:test-fixtures": "pnpm run build:worker && pnpm run build:worker-checksum && node lambda/asset-transform/build.mjs --skip-native",
+    "build:test-fixtures": "pnpm run build:worker && pnpm run build:worker-metadata && node lambda/asset-transform/build.mjs --skip-native",
     "lint": "eslint src/ 'lambda/**/*.ts' 'canary/**/*.ts' 'worker/**/*.ts' 'test-support/**/*.ts'",
     "test": "pnpm run build:test-fixtures && vitest run",
     "typecheck": "tsc --noEmit && tsc --noEmit -p lambda/tsconfig.json && tsc --noEmit -p canary/tsconfig.json && tsc --noEmit -p worker/tsconfig.json && tsc --noEmit -p test-support/tsconfig.json"

package/worker/canopy-worker.service

@@ -26,13 +26,15 @@ SuccessExitStatus=75
 # File output, not journal: the CloudWatch agent cannot read journald,
 # so it tails this file instead (agent config lives in the cms-service.ts
 # user-data, which writes the deployed copy of this unit).
-# CAUTION: /var/log/canopy-worker must exist BEFORE first start. systemd
-# opens append: targets before it creates LogsDirectory= dirs
-# (systemd#27591), so without the pre-created dir the exec fails with
-# 209/STDOUT and Restart=always crash-loops forever. The deployed user-data
-# runs mkdir first; do the same if you install this unit by hand.
-# LogsDirectory= is kept for ownership management on subsequent starts.
-LogsDirectory=canopy-worker
+# systemd opens append: targets as root, following symlinks, before it
+# applies the sandbox or drops to User=. So the worker must own neither the
+# file nor its directory: one that could swap worker.log for a symlink would
+# have its output appended to any file on the host as root. Hence no
+# LogsDirectory=, which chowns the directory to User=. Create both, root-owned,
+# BEFORE the first start (without the directory the exec fails with 209/STDOUT
+# and Restart=always crash-loops):
+#   install -d -o root -g root -m 0755 /var/log/canopy-worker
+#   install -o root -g root -m 0644 /dev/null /var/log/canopy-worker/worker.log
 StandardOutput=append:/var/log/canopy-worker/worker.log
 StandardError=append:/var/log/canopy-worker/worker.log
 
@@ -43,9 +45,12 @@ StateDirectory=canopy-worker
 
 # Environment
 EnvironmentFile=/opt/canopy-worker/.env
+# The worker contract this unit provides (WORKER_CONTRACT_VERSION in
+# src/constructs/worker-lifecycle.ts); a bundle needing more refuses to start.
+Environment=CANOPYCMS_WORKER_CONTRACT=1
 
-# Sandbox. Writable: the workspace, the log dir (LogsDirectory=), the state dir
-# (StateDirectory=) and a private /tmp.
+# Sandbox. Writable: the workspace, the state dir (StateDirectory=) and a
+# private /tmp.
 # ProtectHome=tmpfs, not yes: every git call reads per-user files under $HOME
 # (~/.config/git/attributes, ignore), and an inaccessible /home makes each read
 # warn "Permission denied"; an empty one is a silent ENOENT.

package/worker/dist/index.js

@@ -79169,6 +79169,13 @@ async function repackBareRemoteIfNeeded(gitDir, git = simpleGit().env(gitChildEn
   const after = await countObjects(git, gitDir);
   return { repacked: true, before, after, ms: Date.now() - startedAt };
 }
+function failOnSignalExit(error3, result) {
+  if (error3) return error3;
+  if (result.exitCode === null) {
+    return new Error("git was killed by a signal before it finished");
+  }
+  return void 0;
+}
 var GITHUB_TRACKING_REF_PREFIX = "refs/remotes/github/";
 async function ensureGitExcludePattern(repoPath, pattern) {
   const excludePath = path7.join(repoPath, ".git", "info", "exclude");
@@ -79209,7 +79216,7 @@ init_error();
 init_worker_status();
 
 // ../canopycms/src/version.ts
-var CANOPYCMS_VERSION = "0.0.68-int.110";
+var CANOPYCMS_VERSION = "0.0.68-int.112";
 
 // ../canopycms/src/worker/log.ts
 init_logger();
@@ -79842,6 +79849,27 @@ async function assertOwnDirectory(dir) {
     );
   }
 }
+var RefusedPushError = class extends Error {
+  constructor(branch, reason) {
+    super(
+      reason === "invalid" ? `Refusing to push ${JSON.stringify(branch)} to GitHub: it is not a valid branch name.` : `Refusing to push "${branch}" to GitHub: it is the base branch or GitHub's default branch, which the CanopyCMS worker never pushes. Nothing in CanopyCMS queues such a push; find out what wrote this task.`
+    );
+    this.branch = branch;
+    this.name = "RefusedPushError";
+  }
+};
+async function assertPlainBranchName(branch) {
+  let normalized;
+  try {
+    normalized = (await simpleGit().raw(["check-ref-format", "--branch", branch])).trim();
+  } catch (err) {
+    if (/is not a valid branch name/.test(getErrorMessage(err))) {
+      throw new RefusedPushError(branch, "invalid");
+    }
+    throw err;
+  }
+  if (normalized !== branch) throw new RefusedPushError(branch, "invalid");
+}
 var MirrorSession = class {
   constructor(gitDir, remoteGitPath, timeoutMs) {
     this.gitDir = gitDir;
@@ -79907,6 +79935,21 @@ var MirrorSession = class {
       ...refspecs
     ]);
   }
+  /**
+   * The branch GitHub's HEAD names, or null for a repository with none (empty, or HEAD unborn). A
+   * failure to reach GitHub throws, so a push that cannot learn it does not happen.
+   */
+  async githubDefaultBranch(githubUrl, signal) {
+    const out = await this.git(signal).raw([
+      "ls-remote",
+      "--symref",
+      "--end-of-options",
+      githubUrl,
+      "HEAD"
+    ]);
+    const match2 = /^ref: refs\/heads\/(.+)\tHEAD$/m.exec(out);
+    return match2 ? match2[1] : null;
+  }
   /** The mirror's tip for `branch`, or null when GitHub had no such branch at the last fetch. */
   async branchTip(branch) {
     try {
@@ -79916,6 +79959,26 @@ var MirrorSession = class {
       return null;
     }
   }
+  /**
+   * Whether a branch GitHub had at the last fetch contains the commit `id`, or the commit an
+   * annotated tag `id` points at. False for an object the mirror does not have, or one git cannot
+   * read as a commit.
+   */
+  async isOnGitHub(id) {
+    if (!isObjectId(id)) return false;
+    try {
+      const found = await this.git().raw([
+        "for-each-ref",
+        "--count=1",
+        "--format=%(refname)",
+        `--contains=${id}`,
+        "refs/heads/"
+      ]);
+      return found.trim() !== "";
+    } catch {
+      return false;
+    }
+  }
   /**
    * Copy GitHub's branches into `remote.git`'s tracking namespace, pruning the ones GitHub no
    * longer has. `reconcileTrackedBranches` (worker/git-sync.ts) moves `refs/heads/*` from there.
@@ -79950,10 +80013,18 @@ var MirrorSession = class {
    * commit from `remote.git` first is the step that brings the Lambda's objects across, through
    * the pinned `upload-pack`.
    */
-  async pushToGitHub(githubUrl, branch, sha, options = {}) {
+  async pushToGitHub(githubUrl, branch, sha, options) {
     for (const id of [sha, ...options.lease === void 0 ? [] : [options.lease]]) {
       if (!isObjectId(id)) throw new Error(`Not a commit ID: ${JSON.stringify(id)}`);
     }
+    await assertPlainBranchName(branch);
+    const protectedNames = [...options.protectedBranches];
+    const githubDefault = await this.githubDefaultBranch(githubUrl, options.signal);
+    if (githubDefault !== null) protectedNames.push(githubDefault);
+    const target = sanitizeBranchName(branch);
+    if (protectedNames.some((name) => name === branch || sanitizeBranchName(name) === target)) {
+      throw new RefusedPushError(branch, "protected");
+    }
     const staging = `${STAGING_PREFIX}${branch}`;
     const git = this.git(options.signal);
     try {
@@ -81196,6 +81267,12 @@ function throwIfWorkflowRefusal(branch, message) {
 }
 async function pushBranchToGitHub(ctx, branch, signal) {
   await assertSharedRepoConfig(ctx.remoteGitPath, "bare");
+  try {
+    await assertPlainBranchName(branch);
+  } catch (err) {
+    if (err instanceof RefusedPushError) throw new PermanentTaskError(err.message);
+    throw err;
+  }
   const githubUrl = await ctx.buildGitHubUrl();
   const branchPath = ctx.branchWorkspacePath(branch);
   const metaFile = await BranchMetadataFileManager.loadOnly(branchPath).catch(() => null);
@@ -81205,11 +81282,16 @@ async function pushBranchToGitHub(ctx, branch, signal) {
     throw new Error(`Branch "${branch}" is not in remote.git, so there is nothing to push`);
   }
   const outcome = await ctx.githubMirror().exclusive(async (mirror) => {
-    const push = (lease) => mirror.pushToGitHub(githubUrl, branch, outgoingSha, { lease, signal });
+    const push = (lease) => mirror.pushToGitHub(githubUrl, branch, outgoingSha, {
+      lease,
+      signal,
+      protectedBranches: [ctx.baseBranch]
+    });
     try {
       await push(marker);
       return "pushed";
     } catch (err) {
+      if (err instanceof RefusedPushError) throw new PermanentTaskError(err.message);
       const message = getErrorMessage(err);
       throwIfWorkflowRefusal(branch, message);
       if (marker && isStaleLeaseRejection(message)) {
@@ -82193,7 +82275,10 @@ async function pushSettingsBranches(ctx, git, trackedNames) {
       const sha = (await git.revparse(["--verify", `refs/heads/${settingsBranch}`])).trim();
       const githubUrl = await ctx.buildGitHubUrl();
       await ctx.githubMirror().exclusive(
-        (mirror) => mirror.pushToGitHub(githubUrl, settingsBranch, sha, { signal: ctx.shutdownSignal() })
+        (mirror) => mirror.pushToGitHub(githubUrl, settingsBranch, sha, {
+          signal: ctx.shutdownSignal(),
+          protectedBranches: [ctx.baseBranch]
+        })
       );
       workerLog(`Pushed settings branch ${settingsBranch} to GitHub`);
     } catch (err) {
@@ -82700,6 +82785,29 @@ function defaultStateDirectory(workspacePath) {
   const key = createHash2("sha256").update(path25.resolve(workspacePath)).digest("hex").slice(0, 16);
   return path25.join(os.tmpdir(), `canopycms-worker-${key}`);
 }
+var MAX_REFS_LISTED = 20;
+var RemoteGitKeptError = class extends Error {
+};
+async function listRemoteGitRefs(gitDir) {
+  let stderr = "";
+  const output = await sharedRepoGit(gitDir, "bare", { errors: failOnSignalExit }).outputHandler((_command, _stdout, err) => {
+    err.on("data", (chunk) => {
+      stderr += String(chunk);
+    });
+  }).raw(["for-each-ref", "--format=%(objectname) %(refname)"]);
+  if (stderr.trim() !== "") throw new Error(stderr.trim());
+  const refs = /* @__PURE__ */ new Map();
+  for (const line of output.split("\n")) {
+    const space = line.indexOf(" ");
+    if (space === -1) continue;
+    const ref = line.slice(space + 1);
+    if (!ref.startsWith(GITHUB_TRACKING_REF_PREFIX)) refs.set(ref, line.slice(0, space));
+  }
+  return refs;
+}
+function sameRefs(a9, b10) {
+  return a9.size === b10.size && [...a9].every(([ref, id]) => b10.get(ref) === id);
+}
 var DEFAULT_TASK_TIMEOUT = 6e4;
 var DEFAULT_MAX_RETRIES2 = 3;
 var DEFAULT_LOCK_STALE_MS = 6e4;
@@ -83234,7 +83342,9 @@ var CmsWorker = class {
   /**
    * An unconfigured base branch is the one remote.git's HEAD names, which is what the Lambda
    * reads too (GitManager.detectBaseBranch); before remote.git exists, it is GitHub's default
-   * branch, which the clone then records as that HEAD. Never assumes 'main'.
+   * branch, which the clone then records as that HEAD. So it is too when remote.git has no branch
+   * outside the settings namespace: no base branch is there for HEAD to name, and ensureRemoteGit
+   * replaces it from GitHub unless something in it is at stake. Never assumes 'main'.
    */
   async resolveBaseBranch() {
     if (this.resolvedBaseBranch) return;
@@ -83253,10 +83363,11 @@ var CmsWorker = class {
     } catch (err) {
       throw undetermined(err);
     }
-    if (remoteGitExists) await assertSharedRepoConfig(this.remoteGitPath, "bare");
+    if (remoteGitExists) await this.assertRemoteGitConfig();
     let name;
     try {
-      name = remoteGitExists ? await readHeadBranch(this.remoteGitPath, sharedRepoGit(this.remoteGitPath, "bare")) : (await this.octokitClient().repos.get({
+      const fromRemoteGit = remoteGitExists && await this.hasContentBranch(this.remoteGitPath);
+      name = fromRemoteGit ? await readHeadBranch(this.remoteGitPath, sharedRepoGit(this.remoteGitPath, "bare")) : (await this.octokitClient().repos.get({
         owner: this.config.githubOwner,
         repo: this.config.githubRepo
       })).data.default_branch;
@@ -83266,6 +83377,30 @@ var CmsWorker = class {
     this.setBaseBranch(name);
     workerLog(`Base branch: '${name}' (detected; CANOPYCMS_BASE_BRANCH is not set)`);
   }
+  /**
+   * Check remote.git's config at boot. A refusal is never a case for replacement: it reports that
+   * something wrote there, and listing the refs a replacement would lose means running git under
+   * that config. The message says so, so System health does not read it as a routine failure.
+   */
+  async assertRemoteGitConfig() {
+    try {
+      await assertSharedRepoConfig(this.remoteGitPath, "bare");
+    } catch (err) {
+      if (!(err instanceof SharedRepoRefusalError)) throw err;
+      throw new SharedRepoRefusalError(
+        `${err.message.replace(/\.$/, "")}. This needs an operator: the worker never repairs or replaces a remote.git whose config was tampered with or cannot be read.`
+      );
+    }
+  }
+  /** Whether the bare repo at `gitDir` has a branch that is not a settings branch. */
+  async hasContentBranch(gitDir) {
+    const branches = await sharedRepoGit(gitDir, "bare", { errors: failOnSignalExit }).raw([
+      "for-each-ref",
+      "--format=%(refname:strip=2)",
+      "refs/heads/"
+    ]);
+    return branches.split("\n").some((branch) => branch !== "" && !isSettingsBranchName(branch, this.ensureSettingsBranch()));
+  }
   /**
    * Point remote.git's HEAD at the base branch this worker uses, so a Lambda left to detect it
    * (GitManager.detectBaseBranch) reads the same name. Only the worker writes it, at boot under
@@ -83372,7 +83507,7 @@ var CmsWorker = class {
    * pushed) succeeds and leaves a refs-less repo that `fs.stat` cannot tell from a healthy one, so
    * left unchecked it silently poisons remote.git and the stat short-circuit means it never heals.
    * The base branch is therefore verified right after seeding AND on the already-exists fast
-   * path, since a previous run can have left a poisoned remote.git behind.
+   * path, where a poisoned remote.git is replaced (`replacePoisonedRemoteGit`).
    */
   async ensureRemoteGit() {
     let exists2;
@@ -83383,26 +83518,125 @@ var CmsWorker = class {
       exists2 = false;
     }
     if (exists2) {
-      await assertSharedRepoConfig(this.remoteGitPath, "bare");
+      await this.assertRemoteGitConfig();
       await this.scrubPersistedRemote(this.remoteGitPath);
       try {
         await this.verifyBaseBranchExists(this.remoteGitPath);
       } catch (err) {
         workerLogError(`remote.git base branch verification failed: ${getErrorMessage(err)}`);
-        throw new Error(
-          `remote.git at ${this.remoteGitPath} has no branch '${this.baseBranch}' (likely cloned while the GitHub repo was empty). Delete ${this.remoteGitPath} and restart the worker to re-clone.`
-        );
+        await this.replacePoisonedRemoteGit();
+        return;
       }
       await this.applyRemoteGitConfig(this.remoteGitPath);
       return;
     }
     workerLog("Initializing remote.git from GitHub...");
+    const stagingPath = await this.seedRemoteGitStaging();
+    await fs23.rename(stagingPath, this.remoteGitPath);
+    workerLog("remote.git initialized");
+  }
+  /**
+   * Replace an existing remote.git that has no base branch with a fresh seed from GitHub, but only
+   * when every ref in it names a commit a GitHub branch contains, so replacing it loses no commit;
+   * a branch name GitHub lacks, such as a merged branch deleted there, goes. Otherwise refuse,
+   * naming the refs at stake: unpushed work such as a settings branch.
+   *
+   * The refs are compared in the same mirror session that fetches GitHub and seeds the
+   * replacement, and listed again just before the swap, since the Lambda can push into remote.git
+   * while the seed runs. The swap is two renames, so the name remote.git resolves to the old repo,
+   * the new one, or for an instant nothing, never a half-built or half-deleted one.
+   */
+  async replacePoisonedRemoteGit() {
+    const poisoned = `remote.git at ${this.remoteGitPath} has no branch '${this.baseBranch}' (cloned while the GitHub repository was empty, or the base branch is set to one it never had)`;
+    const notReplaced = (reason) => new Error(`${poisoned}, and it was not replaced: ${reason}`);
+    let refs;
+    try {
+      refs = await listRemoteGitRefs(this.remoteGitPath);
+    } catch (err) {
+      throw notReplaced(
+        `its refs could not all be read (${redactCredentials(getErrorMessage(err)).trim()}), so the worker cannot tell whether replacing it would lose work. Deleting ${this.remoteGitPath} and restarting the worker re-clones it and discards every ref in it.`
+      );
+    }
+    if (refs.has(`refs/heads/${this.baseBranch}`)) {
+      throw new Error(
+        `remote.git at ${this.remoteGitPath} has branch '${this.baseBranch}', but git could not verify it, so the worker will not replace remote.git. Restarting the worker checks again.`
+      );
+    }
+    workerLog(
+      `remote.git has no branch '${this.baseBranch}': replacing it if GitHub has every ref in it`
+    );
+    const stagingPath = await this.seedRemoteGitStaging(async (mirror) => {
+      const unpushed = [];
+      for (const [ref, id] of refs) {
+        if (!await mirror.isOnGitHub(id)) unpushed.push(ref);
+      }
+      if (unpushed.length === 0) return;
+      const shown = unpushed.slice(0, MAX_REFS_LISTED);
+      const more = unpushed.length - shown.length;
+      throw new RemoteGitKeptError(
+        `${poisoned}, and it holds ${unpushed.length === 1 ? "a ref" : `${unpushed.length} refs`} GitHub does not have, so the worker will not replace it: ${shown.join(", ")}${more > 0 ? ` and ${more} more` : ""}. Deleting ${this.remoteGitPath} and restarting the worker re-clones it and discards ${unpushed.length === 1 ? "that ref" : "those refs"}.`
+      );
+    }).catch((err) => {
+      if (err instanceof RemoteGitKeptError) throw err;
+      throw notReplaced(redactCredentials(getErrorMessage(err)));
+    });
+    const discardStaging = () => fs23.rm(stagingPath, { recursive: true, force: true });
+    let current;
+    try {
+      current = await listRemoteGitRefs(this.remoteGitPath);
+    } catch (err) {
+      await discardStaging();
+      throw notReplaced(
+        `its refs could not all be read again (${getErrorMessage(err).trim()}). Restarting the worker checks again.`
+      );
+    }
+    if (!sameRefs(refs, current)) {
+      await discardStaging();
+      throw notReplaced(
+        "its refs changed while the worker seeded its replacement. Restarting the worker checks again."
+      );
+    }
+    const replaced = `${this.remoteGitPath}.replaced-${Date.now()}`;
+    try {
+      await fs23.rename(this.remoteGitPath, replaced);
+    } catch (err) {
+      await discardStaging();
+      throw err;
+    }
+    try {
+      await fs23.rename(stagingPath, this.remoteGitPath);
+    } catch (err) {
+      await fs23.rename(replaced, this.remoteGitPath);
+      await discardStaging();
+      throw err;
+    }
+    const after = await listRemoteGitRefs(replaced).catch(() => null);
+    if (after !== null && sameRefs(refs, after)) {
+      await fs23.rm(replaced, { recursive: true, force: true, maxRetries: 3 }).catch((err) => {
+        workerLogWarn(
+          `Could not remove the replaced remote.git at ${replaced}: ${getErrorMessage(err)}`
+        );
+      });
+    } else {
+      workerLogError(
+        `Kept the replaced remote.git at ${replaced}: its refs changed as it was swapped out, so it may hold work GitHub does not have.`
+      );
+    }
+    workerLog("remote.git replaced");
+  }
+  /**
+   * Seed a new bare repo from GitHub at `<remote.git>.cloning`, verified and configured, and
+   * return its path for the caller to rename into place. `check` runs in the mirror session right
+   * after the GitHub fetch, and its throw stops the seed. Throws with the staging directory gone.
+   */
+  async seedRemoteGitStaging(check) {
     const stagingPath = `${this.remoteGitPath}.cloning`;
     await fs23.rm(stagingPath, { recursive: true, force: true });
     try {
       const githubUrl = await this.buildGitHubUrl();
       await this.githubMirror.exclusive(async (mirror) => {
         await mirror.fetchFromGitHub(githubUrl);
+        await check?.(mirror);
         if (await mirror.branchTip(this.baseBranch) === null) {
           throw new Error(`GitHub has no branch '${this.baseBranch}'`);
         }
@@ -83416,8 +83650,11 @@ var CmsWorker = class {
       await this.recordBaseBranchInRemoteHead(stagingPath);
       await this.applyRemoteGitConfig(stagingPath);
     } catch (err) {
-      workerLogError(`remote.git seeding failed: ${redactCredentials(getErrorMessage(err))}`);
+      if (!(err instanceof RemoteGitKeptError)) {
+        workerLogError(`remote.git seeding failed: ${redactCredentials(getErrorMessage(err))}`);
+      }
       await fs23.rm(stagingPath, { recursive: true, force: true });
+      if (err instanceof RemoteGitKeptError) throw err;
       if (err instanceof SharedRepoRefusalError) {
         throw new SharedRepoRefusalError(
           `Refusing to run git in ${stagingPath}: something wrote to it while the worker seeded remote.git from it` + (err instanceof UntrustedRepoConfigError ? ` (${err.keys.map((k10) => k10.key).join(", ")})` : "") + `, so it was removed. Find out what wrote there; restarting the worker seeds it again.`
@@ -83427,8 +83664,7 @@ var CmsWorker = class {
         `remote.git clone of ${this.config.githubOwner}/${this.config.githubRepo} failed or has no branch '${this.baseBranch}' - the GitHub repository may be empty, or the base branch may not exist. Push an initial commit to '${this.baseBranch}' and restart the worker (systemd will retry automatically).`
       );
     }
-    await fs23.rename(stagingPath, this.remoteGitPath);
-    workerLog("remote.git initialized");
+    return stagingPath;
   }
   /**
    * Refuse a state directory on the shared filesystem: the mirror there would be as writable by
@@ -83863,6 +84099,22 @@ var WORKER_DRAIN_HOOK_NAME = "canopycms-worker-drain";
 var EXIT_DRAINED_FOR_TERMINATION = 75;
 var EXIT_WORKER_SELF_STOPPED = 69;
 var WORKER_CAPACITY_ENV = "CANOPYCMS_WORKER_CAPACITY";
+var WORKER_CONTRACT_REQUIREMENTS = ["StateDirectory=canopy-worker"];
+var WORKER_CONTRACT_VERSION = WORKER_CONTRACT_REQUIREMENTS.length;
+var WORKER_CONTRACT_ENV = "CANOPYCMS_WORKER_CONTRACT";
+function unitWorkerContract(env2) {
+  const stamp = env2[WORKER_CONTRACT_ENV];
+  if (stamp === void 0) return env2.STATE_DIRECTORY ? 1 : 0;
+  return /^\d+$/.test(stamp) ? Number(stamp) : NaN;
+}
+function workerContractShortfall(env2) {
+  const unit = unitWorkerContract(env2);
+  if (Number.isNaN(unit)) {
+    return `canopy-worker: ${WORKER_CONTRACT_ENV}=${JSON.stringify(env2[WORKER_CONTRACT_ENV])} on the worker unit is not a whole number`;
+  }
+  if (unit >= WORKER_CONTRACT_VERSION) return void 0;
+  return `canopy-worker: template too old for this bundle: needs worker contract ${WORKER_CONTRACT_VERSION}, unit has ${unit} (` + WORKER_CONTRACT_REQUIREMENTS.slice(unit).map((setting, i10) => `contract ${unit + i10 + 1} adds ${setting}`).join("; ") + `). Deploy the stack template before rolling this bundle.`;
+}
 
 // worker/termination-watch.ts
 var DEFAULT_IMDS_ENDPOINT = "http://169.254.169.254";
@@ -93472,6 +93724,8 @@ async function runWorker(deps) {
     workerLog("CMS Worker starting...");
     workspacePath = env2.CANOPYCMS_WORKSPACE_ROOT;
     if (!workspacePath) throw new Error("CANOPYCMS_WORKSPACE_ROOT is required");
+    const contractShortfall = workerContractShortfall(env2);
+    if (contractShortfall) throw new Error(contractShortfall);
     const githubOwner = env2.CANOPYCMS_GITHUB_OWNER;
     if (!githubOwner) throw new Error("CANOPYCMS_GITHUB_OWNER is required");
     const githubRepo = env2.CANOPYCMS_GITHUB_REPO;

package/worker/dist/index.js.contract

@@ -0,0 +1 @@
+1

package/worker/dist/index.js.sha256

@@ -1 +1 @@
-e039d186388e604a02d5271caa938f20b07676d232bbd96665f4e4bd5dd8eaa7  index.js
+7d6f470af88ec0126d5c10e5056cccc4a2ef52e3a464a4fee6a613b4f1583b7f  index.js