hecks 3.6.0 → 3.7.0

data/lib/hecks/projections/deploy/box/settings/types.rb

@@ -51,10 +51,14 @@ module Hecks
               :storage_gb, :backup_days, :snapshots_keep, :database_name, :engine_version,
               :containers, :routes, :default_container, :origin_header, :origin_secret,
               :secret_prefixes, :writable_secrets, :origin_env, :tunnel, :tunnel_service, :proxy_image,
-              :task_definition, :migration, :s3_buckets, :hosting, keyword_init: true
+              :task_definition, :migration, :s3_buckets, :hosting, :shared_database, keyword_init: true
             ) do
-              # @return [String] the CloudFormation stack that holds the database
-              def rds_stack = "#{stack_prefix}-#{infra_name}-rds"
+              # @return [String] the CloudFormation stack that holds the database: the shared
+              #   instance's when the world names one, else this site's own
+              def rds_stack = shared_database || "#{stack_prefix}-#{infra_name}-rds"
+
+              # @return [Boolean] whether the database lives on an instance shared with other sites
+              def shared? = !shared_database.nil?
 
               # @return [String] the CloudFormation stack that holds the box
               def box_stack = "#{stack_prefix}-#{infra_name}-box"

data/lib/hecks/projections/deploy/box/settings.rb

@@ -47,6 +47,7 @@ module Hecks
           def extra_fields(settings, task_definition, database_name)
             {
               task_definition: task_definition, migration: read_migration(settings[:migration], database_name),
+              shared_database: read_shared_database(settings),
               s3_buckets: read_s3_access(settings.fetch(:s3_access, [])),
               hosting: HostingSettings.read(settings, task_definition)
             }
@@ -129,6 +130,23 @@ module Hecks
             end.uniq(&:name)
           end
 
+          # The stack of the RDS instance this site shares with others. The instance's size,
+          # storage, engine and backups are the instance's, so a world that sets them is refused.
+          #
+          # @param settings [Hash{Symbol => Object}] the world's `AwsBox` settings
+          # @return [String, nil] the shared stack's name, or nil for a database of its own
+          # @raise [ArgumentError] when the name is malformed or the world also sizes the instance
+          def read_shared_database(settings)
+            return nil unless settings.key?(:shared_database)
+
+            clash = SHARED_INSTANCE_SETTINGS & settings.keys
+            unless clash.empty?
+              raise ArgumentError, "shared_database: #{clash.join(", ")} size the instance, which belongs to the shared " \
+                                   "stack #{settings[:shared_database]}; drop #{clash.size == 1 ? "it" : "them"}"
+            end
+            check(:shared_database, settings[:shared_database], NAME)
+          end
+
           def read_database_name(settings, infra_name)
             check(:database_name, settings.fetch(:database_name, infra_name.gsub(/[^a-zA-Z0-9]/, "")), DB_NAME)
           end

data/lib/hecks/projections/deploy/box/shared_database.rb

@@ -0,0 +1,69 @@
+module Hecks
+  module Projections
+    module Deploy
+      module Box
+        # The parts of a box deploy that change when its database is one of several on a shared RDS
+        # instance, each client in its own database with its own login role (ADR 0092). Mixed into
+        # `Box`. Everything here returns the dedicated-instance text unchanged when the plan has no
+        # `shared_database`, so a site with its own instance generates the same files as before.
+        module SharedDatabase
+          # @param plan [Settings::Plan] the resolved settings
+          # @return [Hash{String => String}] the Makefile markers that depend on where the database
+          #   lives
+          def database_makefile_values(plan)
+            return dedicated_makefile_values unless plan.shared?
+
+            { "WHAT" => "one app box and a database on the shared instance #{plan.rds_stack}",
+              "REHEARSAL_NOTE" => "# true gives the box no Elastic IP. It never deletes the shared instance\n" \
+                                  "# or this site's database.\n",
+              "RDS_DEPLOY" => "", "DB_SECRET_PARAM" => "\t\tDbSecretArn=$$(aws secretsmanager describe-secret --secret-id " \
+                                                       "#{plan.infra_name}/database --query ARN --output text) \\",
+              "PHONY" => ".PHONY: stacks deploy provision", "PROVISION" => provision_target(plan) }
+          end
+
+          # @param plan [Settings::Plan] the resolved settings
+          # @return [String] the line of `deploy-box.sh` that finds the secret the containers read
+          def database_secret_line(plan)
+            return 'DB_SECRET=$(out "$RDS_STACK" DbSecretArn)' unless plan.shared?
+
+            "DB_SECRET=$(aws secretsmanager describe-secret --secret-id #{plan.infra_name}/database --query ARN --output text)"
+          end
+
+          # @param plan [Settings::Plan] the resolved settings
+          # @return [String] `provision-database.sh`, which creates the site's database, role and
+          #   secret
+          def provision_database_sh(plan)
+            template("provision-database.sh.tmpl", "STACK" => plan.infra_name, "RDS_STACK" => plan.rds_stack,
+                                                   "DATABASE" => plan.database_name)
+          end
+
+          private
+
+          def dedicated_makefile_values
+            { "WHAT" => "one app box and one RDS instance",
+              "REHEARSAL_NOTE" => "# true makes a throwaway pair: the database is deleted with its stack and the box has no\n" \
+                                  "# Elastic IP. The default is a production pair, with deletion protection.\n",
+              "RDS_DEPLOY" => rds_deploy_lines, "DB_SECRET_PARAM" => dedicated_secret_param,
+              "PHONY" => ".PHONY: stacks deploy", "PROVISION" => "" }
+          end
+
+          def rds_deploy_lines
+            "\taws cloudformation deploy --template-file rds.yaml --stack-name $(RDS_STACK) --capabilities CAPABILITY_IAM \\\n" \
+              "\t\t--parameter-overrides VpcId=$(VPC) PrivateSubnetIds=$(PRIVATE_SUBNETS) Rehearsal=$(REHEARSAL)\n"
+          end
+
+          def dedicated_secret_param
+            "\t\tDbSecretArn=$$(aws cloudformation describe-stacks --stack-name $(RDS_STACK) " \
+              "--query \"Stacks[0].Outputs[?OutputKey=='DbSecretArn'].OutputValue\" --output text) \\"
+          end
+
+          def provision_target(plan)
+            "# Creates #{plan.infra_name}'s database, role and secret on #{plan.rds_stack}. Run it once; run it again with\n" \
+              "# ROTATE=--rotate to change the role's password.  make provision BASTION=i-... [ROTATE=--rotate]\n" \
+              "provision:\n\tbash ./provision-database.sh $(BASTION) $(ROTATE)\n\n"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/hecks/projections/deploy/box/templates/Makefile.tmpl

@@ -1,24 +1,23 @@
-# @@STACK@@: one app box and one RDS instance.
+# @@STACK@@: @@WHAT@@.
 #   make stacks VPC=vpc-... PRIVATE_SUBNETS=subnet-a,subnet-b PUBLIC_SUBNET=subnet-c [REHEARSAL=true]
 #   make deploy @@DEPLOY_HINT@@
 RDS_STACK = @@RDS_STACK@@
 BOX_STACK = @@BOX_STACK@@
-# true makes a throwaway pair: the database is deleted with its stack and the box has no
-# Elastic IP. The default is a production pair, with deletion protection.
+@@REHEARSAL_NOTE@@
 REHEARSAL ?= false
 
 HECKS ?= hecks
 
-.PHONY: stacks deploy
+@@PHONY@@
 stacks:
-	aws cloudformation deploy --template-file rds.yaml --stack-name $(RDS_STACK) --capabilities CAPABILITY_IAM \
-		--parameter-overrides VpcId=$(VPC) PrivateSubnetIds=$(PRIVATE_SUBNETS) Rehearsal=$(REHEARSAL)
+@@RDS_DEPLOY@@
 	aws cloudformation deploy --template-file box.yaml --stack-name $(BOX_STACK) --capabilities CAPABILITY_IAM \
 		--parameter-overrides VpcId=$(VPC) SubnetId=$(PUBLIC_SUBNET) \
 		DbSecurityGroupId=$$(aws cloudformation describe-stacks --stack-name $(RDS_STACK) --query "Stacks[0].Outputs[?OutputKey=='DbSecurityGroupId'].OutputValue" --output text) \
-		DbSecretArn=$$(aws cloudformation describe-stacks --stack-name $(RDS_STACK) --query "Stacks[0].Outputs[?OutputKey=='DbSecretArn'].OutputValue" --output text) \
+@@DB_SECRET_PARAM@@
 		AlertTopicArn=$$(aws cloudformation describe-stacks --stack-name $(RDS_STACK) --query "Stacks[0].Outputs[?OutputKey=='AlertTopicArn'].OutputValue" --output text) \
 		Rehearsal=$(REHEARSAL)
 
+@@PROVISION@@
 deploy:
 @@DEPLOY_RECIPE@@

data/lib/hecks/projections/deploy/box/templates/deploy-box.sh.tmpl

@@ -26,7 +26,7 @@ out() { aws cloudformation describe-stacks --stack-name "$1" --query "Stacks[0].
 
 BOX=$(out "$BOX_STACK" InstanceId)
 DB_HOST=$(out "$RDS_STACK" DbEndpoint)
-DB_SECRET=$(out "$RDS_STACK" DbSecretArn)
+@@DB_SECRET_LINE@@
 [ -n "$BOX" ] && [ "$BOX" != None ] || { echo "no instance in stack $BOX_STACK" >&2; exit 40; }
 ACCOUNT=$(aws sts get-caller-identity --query Account --output text)
 

data/lib/hecks/projections/deploy/box/templates/provision-database.sh.tmpl

@@ -0,0 +1,97 @@
+#!/bin/bash
+# Create @@STACK@@'s database and login role on the shared RDS instance (stack @@RDS_STACK@@), and the
+# secret @@STACK@@/database that the box's containers read. Run once for a new site, from a machine with
+# AWS credentials that may read the instance's master secret and start SSM sessions, through a bastion
+# that can reach the instance.
+#
+#   provision-database.sh <bastion-instance-id> [--rotate]
+#
+# The role owns only the database @@DATABASE@@, is not a superuser, and no other role may connect to that
+# database. The secret holds {username, password, host, port, dbname}; the box's role reads that secret
+# and never the instance's master secret.
+#
+# Safe to run again: a role and a database that exist are left alone, and a secret that exists keeps
+# its password. --rotate gives the role a new password and writes it to the secret; roll the box after it.
+#
+# Ends 70 (the shared stack has no instance).
+#
+# Needs psql, jq, openssl, the SSM plugin and AWS credentials.
+set -euo pipefail
+
+BASTION=${1:?bastion instance id}
+ROTATE=${2:-}
+[ -z "$ROTATE" ] || [ "$ROTATE" = --rotate ] || { echo "usage: provision-database.sh <bastion-instance-id> [--rotate]" >&2; exit 2; }
+RDS_STACK=@@RDS_STACK@@
+DB=@@DATABASE@@
+ROLE=@@DATABASE@@
+SECRET=@@STACK@@/database
+PORT=15434
+out() { aws cloudformation describe-stacks --stack-name "$1" --query "Stacks[0].Outputs[?OutputKey==\`$2\`].OutputValue" --output text; }
+
+HOST=$(out "$RDS_STACK" DbEndpoint)
+MASTER=$(out "$RDS_STACK" DbSecretArn)
+[ -n "$HOST" ] && [ "$HOST" != None ] || { echo "no instance in stack $RDS_STACK" >&2; exit 70; }
+
+WORK=$(mktemp -d)
+PIDS=()
+cleanup() { for p in "${PIDS[@]:-}"; do [ -n "$p" ] && kill "$p" 2>/dev/null || true; done; rm -rf "$WORK"; }
+trap cleanup EXIT
+
+master_field() { aws secretsmanager get-secret-value --secret-id "$MASTER" --query SecretString --output text | jq -r "$1"; }
+MASTER_USER=$(master_field '.username // "postgres"')
+MASTER_PW=$(master_field .password)
+
+aws ssm start-session --target "$BASTION" --document-name AWS-StartPortForwardingSessionToRemoteHost \
+  --parameters "{\"host\":[\"$HOST\"],\"portNumber\":[\"5432\"],\"localPortNumber\":[\"$PORT\"]}" >"$WORK/tunnel.log" 2>&1 &
+PIDS+=($!)
+disown
+for _ in $(seq 1 20); do grep -q "Waiting for connections" "$WORK/tunnel.log" 2>/dev/null && break; sleep 1; done
+
+# One statement, one answer. The master connects to the maintenance database.
+M() { PGPASSWORD=$MASTER_PW psql -h localhost -p $PORT -U "$MASTER_USER" -d postgres -v ON_ERROR_STOP=1 -Atc "$1"; }
+# A script on stdin, with the new password passed as a variable so it is never in a command line.
+MS() { PGPASSWORD=$MASTER_PW psql -h localhost -p $PORT -U "$MASTER_USER" -d postgres -v ON_ERROR_STOP=1 -v pw="$PW" -q -f -; }
+
+HAVE_ROLE=$(M "select count(*) from pg_roles where rolname='$ROLE'")
+HAVE_SECRET=no
+aws secretsmanager describe-secret --secret-id "$SECRET" >/dev/null 2>&1 && HAVE_SECRET=yes
+
+# Letters and digits only: the host puts the password in a URL without percent-encoding it.
+PW=""
+if [ "$HAVE_ROLE" = 0 ]; then
+  PW=$(openssl rand -hex 16)
+  MS <<SQL
+create role "$ROLE" login password :'pw' nosuperuser nocreatedb nocreaterole noinherit;
+SQL
+  echo "created role $ROLE"
+elif [ "$ROTATE" = --rotate ] || [ "$HAVE_SECRET" = no ]; then
+  PW=$(openssl rand -hex 16)
+  MS <<SQL
+alter role "$ROLE" password :'pw';
+SQL
+  echo "set a new password for role $ROLE"
+else
+  echo "role $ROLE exists and so does $SECRET; kept"
+fi
+
+# RDS's master is not a true superuser: it may create a database for a role only as a member of that role.
+if [ "$(M "select count(*) from pg_database where datname='$DB'")" = 0 ]; then
+  M "grant \"$ROLE\" to \"$MASTER_USER\"" >/dev/null
+  M "create database \"$DB\" owner \"$ROLE\"" >/dev/null
+  echo "created database $DB"
+else
+  echo "database $DB exists"
+fi
+# Nobody but the owner (and the master) may connect to it.
+M "revoke all on database \"$DB\" from public" >/dev/null
+
+if [ -n "$PW" ]; then
+  BODY=$(jq -n --arg u "$ROLE" --arg p "$PW" --arg h "$HOST" --arg d "$DB" '{username: $u, password: $p, host: $h, port: 5432, dbname: $d}')
+  if [ "$HAVE_SECRET" = yes ]; then
+    aws secretsmanager put-secret-value --secret-id "$SECRET" --secret-string "$BODY" >/dev/null
+  else
+    aws secretsmanager create-secret --name "$SECRET" --description "@@STACK@@'s database login on @@RDS_STACK@@" --secret-string "$BODY" >/dev/null
+  fi
+  echo "wrote secret $SECRET"
+fi
+echo "ok   $DB on $HOST, role $ROLE, secret $SECRET"

data/lib/hecks/projections/deploy/box/templates/restore-to-rds.sh.tmpl

@@ -7,6 +7,8 @@
 #
 #   FORCE=1   drop the target schemas first: a re-load before cutover, or a rollback copy in the other
 #             direction (swap the hosts and secrets, and SRC_DB and DST_DB)
+#   SCHEMAS   the schemas to copy, space separated (default @@SCHEMAS@@); another database's schemas, such as
+#             an analytics database's `public`, are copied by naming them here with SRC_DB and DST_DB
 #   SRC_DB    the source database (default @@SOURCE_DATABASE@@)
 #   DST_DB    the target database (default @@DATABASE@@)
 #   VERIFY_BY_COMMAND=1   end after the copy and leave the comparison to `hecks deploy data_copy.restore`
@@ -28,7 +30,7 @@ HERE=$(cd "$(dirname "$0")" && pwd)
 
 BASTION=${1:?bastion instance id}; SRC_HOST=${2:?source host}; SRC_SECRET=${3:?source secret arn}
 DST_HOST=${4:?target host}; DST_SECRET=${5:?target secret arn}
-SCHEMAS="@@SCHEMAS@@"
+SCHEMAS=${SCHEMAS:-"@@SCHEMAS@@"}
 SRC_DB=${SRC_DB:-@@SOURCE_DATABASE@@}; DST_DB=${DST_DB:-@@DATABASE@@}
 SRC_PORT=15432; DST_PORT=15433
 
@@ -93,4 +95,4 @@ done
 
 # Same structure, same exact row counts in every table, nothing unpopulated.
 [ -z "${VERIFY_BY_COMMAND:-}" ] || exit 0
-A_DB=$SRC_DB B_DB=$DST_DB bash "$HERE/verify-copy.sh" "$BASTION" "$SRC_HOST" "$SRC_SECRET" "$DST_HOST" "$DST_SECRET"
+A_DB=$SRC_DB B_DB=$DST_DB COMPARE_SCHEMAS=$(echo "$SCHEMAS" | tr ' ' ',') bash "$HERE/verify-copy.sh" "$BASTION" "$SRC_HOST" "$SRC_SECRET" "$DST_HOST" "$DST_SECRET"

data/lib/hecks/projections/deploy/box/templates/verify-copy.sh.tmpl

@@ -6,13 +6,14 @@
 #   verify-copy.sh <bastion-instance-id> <host-a> <secret-arn-a> <host-b> <secret-arn-b>
 #
 #   A_DB, B_DB   the database on each side (default @@DATABASE@@)
+#   COMPARE_SCHEMAS   the schemas to compare, comma separated (default @@SCHEMAS_CSV@@)
 #
 # Schemas compared: @@SCHEMAS@@
 # Exit 0 only if both are identical and no materialized view is unpopulated in either; 50 when they differ.
 set -euo pipefail
 
 BASTION=${1:?bastion instance id}; A_HOST=${2:?}; A_SECRET=${3:?}; B_HOST=${4:?}; B_SECRET=${5:?}
-SCHEMAS="@@SCHEMAS_CSV@@"
+SCHEMAS=${COMPARE_SCHEMAS:-"@@SCHEMAS_CSV@@"}
 A_DB=${A_DB:-@@DATABASE@@}; B_DB=${B_DB:-@@DATABASE@@}
 A_PORT=15434; B_PORT=15435
 

data/lib/hecks/projections/deploy/box.rb

@@ -7,6 +7,7 @@ require_relative "box/caddy"
 require_relative "box/access"
 require_relative "box/stack_yaml"
 require_relative "box/compose"
+require_relative "box/shared_database"
 require_relative "box/roll_scripts"
 
 module Hecks
@@ -22,6 +23,7 @@ module Hecks
         extend Access
         extend StackYaml
         extend Compose
+        extend SharedDatabase
         extend RollScripts
 
         projects_as :aws_box, needs_world: true, emits: :files
@@ -129,13 +131,23 @@ module Hecks
         # @return [Hash{String => String}] every generated file
         def render_all(plan, region)
           {
-            "rds.yaml" => rds_yaml(plan), "box.yaml" => box_yaml(plan, region),
+            "box.yaml" => box_yaml(plan, region),
             "Caddyfile" => caddyfile(plan), "services.json" => services_json(plan),
             "render-compose.sh" => render_compose_sh(plan, region),
             "fetch-secrets.sh" => File.read(File.join(TEMPLATE_DIR, "fetch-secrets.sh")),
             "deploy-box.sh" => deploy_box_sh(plan, region),
             "Makefile" => makefile(plan)
-          }.merge(migration_files(plan)).then { |files| Hosting.extend_files(files, plan: plan, region: region) }
+          }.merge(database_files(plan)).merge(migration_files(plan))
+            .then { |files| Hosting.extend_files(files, plan: plan, region: region) }
+        end
+
+        # @param plan [Settings::Plan] the resolved settings
+        # @return [Hash{String => String}] the database stack, or for a database on a shared
+        #   instance the script that provisions it; the shared stack is not this site's to generate
+        def database_files(plan)
+          return { "rds.yaml" => rds_yaml(plan) } unless plan.shared?
+
+          { "provision-database.sh" => provision_database_sh(plan) }
         end
 
         # Fills `@@NAME@@` markers. A marker alone on its line is replaced together with the line,

data/lib/hecks/projections/deploy/shared_database_instance/templates/Makefile.tmpl

@@ -0,0 +1,11 @@
+# @@STACK@@: the shared RDS instance that each site's own box stack keeps a database on.
+#   make stack VPC=vpc-... PRIVATE_SUBNETS=subnet-a,subnet-b [BASTION_SG=sg-...] [ALERT_EMAIL=you@example.com] [REHEARSAL=true]
+STACK = @@STACK@@
+# true makes a throwaway copy: no deletion protection and no final snapshot. The default is the real instance.
+REHEARSAL ?= false
+
+.PHONY: stack
+stack:
+	aws cloudformation deploy --template-file rds.yaml --stack-name $(STACK) --capabilities CAPABILITY_IAM \
+		--parameter-overrides VpcId=$(VPC) PrivateSubnetIds=$(PRIVATE_SUBNETS) Rehearsal=$(REHEARSAL) \
+		$(if $(BASTION_SG),BastionSecurityGroupId=$(BASTION_SG)) $(if $(ALERT_EMAIL),AlertEmail=$(ALERT_EMAIL))

data/lib/hecks/projections/deploy/shared_database_instance/templates/README.md.tmpl

@@ -0,0 +1,25 @@
+# @@STACK@@
+
+The shared RDS Postgres instance (@@DATABASE_CLASS@@, @@STORAGE_GB@@ GB, Postgres @@ENGINE_VERSION@@, @@BACKUP_DAYS@@-day backups) on which several
+sites each keep their own database and login role. Generated by `hecks deploy project`; do not edit `rds.yaml`.
+
+## Create it
+
+```
+make stack VPC=vpc-... PRIVATE_SUBNETS=subnet-a,subnet-b BASTION_SG=sg-... ALERT_EMAIL=you@example.com
+```
+
+`BASTION_SG` is the security group of the machine that tunnels to the instance over SSM (a site's own box works).
+Outputs: `DbEndpoint`, `DbSecurityGroupId`, `DbSecretArn` (the master secret) and `AlertTopicArn`.
+
+## Add a site
+
+A site's world says `shared_database "@@STACK@@"`. Its generated `provision-database.sh` creates that site's role, its
+database, and the secret `<site>/database` that its containers read; its box stack adds its own ingress rule to this
+instance's security group. Nothing else needs to change here.
+
+## Things to know
+
+- The master secret is read only by the person who provisions a site, never by a box.
+- This stack cannot be deleted while any site's box stack still references its security group.
+- One maintenance window, restart or failure touches every site. Alarms (storage, CPU, memory) are for the instance.

data/lib/hecks/projections/deploy/shared_database_instance/templates/rds.yaml.tmpl

@@ -0,0 +1,180 @@
+AWSTemplateFormatVersion: "2010-09-09"
+Description: >
+  The shared RDS Postgres instance @@STACK@@, on which several sites each keep their own database and
+  login role. Private subnets only; each site's box stack admits itself on 5432, so nothing else can reach
+  it. The master password is managed by RDS in Secrets Manager ({username, password}); only the operator
+  who runs a site's provision-database.sh reads it, and no box does.
+
+Parameters:
+  VpcId:
+    Type: AWS::EC2::VPC::Id
+  BastionSecurityGroupId:
+    Type: String
+    Default: ""
+    Description: >
+      A security group allowed to reach the instance on 5432, for the bastion that provision-database.sh and
+      restore-to-rds.sh tunnel through. Blank admits nothing but the sites' boxes.
+  PrivateSubnetIds:
+    Type: List<AWS::EC2::Subnet::Id>
+    Description: At least two subnets in different availability zones (an RDS subnet group needs them).
+  InstanceClass:
+    Type: String
+    Default: @@DB_CLASS@@
+  AllocatedStorageGb:
+    Type: Number
+    Default: @@STORAGE_GB@@
+  EngineVersion:
+    Type: String
+    Default: "@@ENGINE_VERSION@@"
+    Description: A major version ("16") lets RDS pick its current minor release.
+  BackupRetentionDays:
+    Type: Number
+    Default: @@BACKUP_DAYS@@
+  ExistingAlertTopicArn:
+    Type: String
+    Default: ""
+    Description: An SNS topic that already has subscribers. Blank creates a new topic.
+  AlertEmail:
+    Type: String
+    Default: ""
+    Description: Subscribed to a new topic (AWS sends one confirmation link). Ignored when ExistingAlertTopicArn is set.
+  Rehearsal:
+    Type: String
+    Default: "false"
+    AllowedValues: ["true", "false"]
+    Description: >
+      true is a throwaway copy (no deletion protection, no final snapshot). false is the real
+      database: deletion protection on and a snapshot taken on delete.
+
+Conditions:
+  NewTopic: !Equals [!Ref ExistingAlertTopicArn, ""]
+  NewTopicWithEmail: !And [!Condition NewTopic, !Not [!Equals [!Ref AlertEmail, ""]]]
+  IsRehearsal: !Equals [!Ref Rehearsal, "true"]
+  HasBastion: !Not [!Equals [!Ref BastionSecurityGroupId, ""]]
+
+Resources:
+  # No inline rules: every site's box stack adds its own ingress rule to this group, and CloudFormation can
+  # revoke an externally added rule when it updates an inline list. The bastion's rule is its own resource.
+  DbSecurityGroup:
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      GroupDescription: @@STACK@@ RDS, 5432 from each site's box (their stacks add the rules) and a bastion if given
+      VpcId: !Ref VpcId
+
+  BastionIngress:
+    Type: AWS::EC2::SecurityGroupIngress
+    Condition: HasBastion
+    Properties:
+      GroupId: !Ref DbSecurityGroup
+      IpProtocol: tcp
+      FromPort: 5432
+      ToPort: 5432
+      SourceSecurityGroupId: !Ref BastionSecurityGroupId
+      Description: Bastion for provision-database.sh and restore-to-rds.sh
+
+  DbSubnetGroup:
+    Type: AWS::RDS::DBSubnetGroup
+    Properties:
+      DBSubnetGroupDescription: @@STACK@@ RDS subnets
+      SubnetIds: !Ref PrivateSubnetIds
+
+  Database:
+    Type: AWS::RDS::DBInstance
+    DeletionPolicy: !If [IsRehearsal, Delete, Snapshot]
+    UpdateReplacePolicy: !If [IsRehearsal, Delete, Snapshot]
+    Properties:
+      Engine: postgres
+      EngineVersion: !Ref EngineVersion
+      DBInstanceClass: !Ref InstanceClass
+      AllocatedStorage: !Ref AllocatedStorageGb
+      StorageType: gp3
+      StorageEncrypted: true
+      MasterUsername: postgres
+      ManageMasterUserPassword: true
+      DBSubnetGroupName: !Ref DbSubnetGroup
+      VPCSecurityGroups: [!Ref DbSecurityGroup]
+      PubliclyAccessible: false
+      MultiAZ: false
+      BackupRetentionPeriod: !Ref BackupRetentionDays
+      DeletionProtection: !If [IsRehearsal, false, true]
+      AutoMinorVersionUpgrade: true
+      CopyTagsToSnapshot: true
+      Tags:
+        - Key: Name
+          Value: @@STACK@@-rds
+
+  AlertTopic:
+    Type: AWS::SNS::Topic
+    Condition: NewTopic
+    Properties:
+      DisplayName: @@STACK@@-alerts
+
+  AlertSubscription:
+    Type: AWS::SNS::Subscription
+    Condition: NewTopicWithEmail
+    Properties:
+      TopicArn: !Ref AlertTopic
+      Protocol: email
+      Endpoint: !Ref AlertEmail
+
+  # Every site's journal only grows, and RDS stops accepting writes when storage is full.
+  StorageLowAlarm:
+    Type: AWS::CloudWatch::Alarm
+    Properties:
+      AlarmDescription: RDS free storage under 3 GB
+      Namespace: AWS/RDS
+      MetricName: FreeStorageSpace
+      Dimensions: [{ Name: DBInstanceIdentifier, Value: !Ref Database }]
+      Statistic: Minimum
+      Period: 300
+      EvaluationPeriods: 3
+      Threshold: 3221225472
+      ComparisonOperator: LessThanThreshold
+      TreatMissingData: breaching
+      AlarmActions: [!If [NewTopic, !Ref AlertTopic, !Ref ExistingAlertTopicArn]]
+      OKActions: [!If [NewTopic, !Ref AlertTopic, !Ref ExistingAlertTopicArn]]
+
+  CpuHighAlarm:
+    Type: AWS::CloudWatch::Alarm
+    Properties:
+      AlarmDescription: "RDS CPU above 80% for 15 minutes (a burstable class may be out of credits)"
+      Namespace: AWS/RDS
+      MetricName: CPUUtilization
+      Dimensions: [{ Name: DBInstanceIdentifier, Value: !Ref Database }]
+      Statistic: Average
+      Period: 300
+      EvaluationPeriods: 3
+      Threshold: 80
+      ComparisonOperator: GreaterThanThreshold
+      TreatMissingData: notBreaching
+      AlarmActions: [!If [NewTopic, !Ref AlertTopic, !Ref ExistingAlertTopicArn]]
+      OKActions: [!If [NewTopic, !Ref AlertTopic, !Ref ExistingAlertTopicArn]]
+
+  MemoryLowAlarm:
+    Type: AWS::CloudWatch::Alarm
+    Properties:
+      AlarmDescription: RDS freeable memory under 100 MB for 15 minutes
+      Namespace: AWS/RDS
+      MetricName: FreeableMemory
+      Dimensions: [{ Name: DBInstanceIdentifier, Value: !Ref Database }]
+      Statistic: Minimum
+      Period: 300
+      EvaluationPeriods: 3
+      Threshold: 104857600
+      ComparisonOperator: LessThanThreshold
+      TreatMissingData: notBreaching
+      AlarmActions: [!If [NewTopic, !Ref AlertTopic, !Ref ExistingAlertTopicArn]]
+      OKActions: [!If [NewTopic, !Ref AlertTopic, !Ref ExistingAlertTopicArn]]
+
+Outputs:
+  AlertTopicArn:
+    Description: Pass as AlertTopicArn to the box stack.
+    Value: !If [NewTopic, !Ref AlertTopic, !Ref ExistingAlertTopicArn]
+  DbEndpoint:
+    Value: !GetAtt Database.Endpoint.Address
+  DbSecretArn:
+    Description: RDS-managed master secret ({username, password}).
+    Value: !GetAtt Database.MasterUserSecret.SecretArn
+  DbSecurityGroupId:
+    Description: Pass as DbSecurityGroupId to the box stack.
+    Value: !Ref DbSecurityGroup