upright 0.3.0 → 0.4.0

data/lib/generators/upright/install/templates/deploy.yml

@@ -48,6 +48,7 @@ env:
   secret:
     - RAILS_MASTER_KEY
     - PROMETHEUS_OTLP_TOKEN
+    - ADMIN_PASSWORD
   tags:
     london:
       SITE_SUBDOMAIN: lon
@@ -62,7 +63,13 @@ volumes:
 
 accessories:
   otel_collector:
-    image: otel/opentelemetry-collector-contrib:0.126.0
+    # Pinned by digest so deploys are immutable; the tag is documentation only.
+    image: otel/opentelemetry-collector-contrib:0.126.0@sha256:973747f78266a8ffec428417727e6b704559e9a30e4be8c1cca2899facd68689
+    # The collector runs unprivileged and only receives OTLP over the private
+    # Docker network — it does not mount the Docker socket or host log
+    # directories. To collect container logs, prefer a filelog receiver over
+    # a read-only mount of the specific log path rather than the Docker socket,
+    # which grants root-equivalent control of the host.
     env:
       secret:
         - OTEL_GATEWAY_USERNAME
@@ -71,17 +78,6 @@ accessories:
         - PROMETHEUS_OTLP_TOKEN
     files:
       - config/otel_collector.yml:/etc/otelcol-contrib/config.yaml
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-      - /var/lib/docker/containers:/var/lib/docker/containers
-    options:
-      user: 0
-    roles:
-      - jobs
-
-  playwright:
-    image: jacoblincool/playwright:chromium-server-1.55.0
-    port: "127.0.0.1:53333:53333"
     roles:
       - jobs
 
@@ -89,12 +85,16 @@ accessories:
     image: prom/prometheus:v3.2.1
     hosts:
       - <%= app_domain %>
+    # No published port: Prometheus is only reachable over the private Docker
+    # network (the app proxies it). --web.enable-lifecycle is deliberately
+    # omitted — the unauthenticated reload/quit endpoints aren't needed since
+    # config changes redeploy the accessory; if you add it back, front it with
+    # --web.config.file authentication.
     cmd: >-
       --config.file=/etc/prometheus/prometheus.yml
       --storage.tsdb.path=/prometheus
       --storage.tsdb.retention.time=30d
       --web.enable-otlp-receiver
-      --web.enable-lifecycle
     files:
       - config/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
       - config/prometheus/rules/upright.rules.yml:/etc/prometheus/rules/upright.rules.yml

data/lib/generators/upright/install/templates/development_prometheus.yml

@@ -9,4 +9,4 @@ scrape_configs:
   - job_name: "upright"
     scrape_interval: 5s
     static_configs:
-      - targets: ["localhost:9394"]
+      - targets: ["host.docker.internal:9394"]

data/lib/generators/upright/install/templates/docker-compose.yml

@@ -2,7 +2,14 @@ services:
   prometheus:
     image: prom/prometheus:v3.4.1
     container_name: <%= app_name %>-prometheus
-    network_mode: host
+    # Published on loopback only, so the dev server is not reachable from other
+    # machines. host.docker.internal lets Prometheus scrape the app running on
+    # the host, on Linux as well as on macOS and Windows Docker Desktop, where
+    # network_mode: host does not reach the host's localhost.
+    ports:
+      - "127.0.0.1:9090:9090"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     volumes:
       - ./config/prometheus/development/prometheus.yml:/etc/prometheus/prometheus.yml:ro
       - ./config/prometheus/rules:/etc/prometheus/rules:ro
@@ -10,14 +17,16 @@ services:
     command:
       - '--config.file=/etc/prometheus/prometheus.yml'
       - '--storage.tsdb.path=/prometheus'
-      - '--web.enable-lifecycle'
+      # --web.enable-lifecycle is omitted: its unauthenticated reload and quit
+      # endpoints are not needed (restart the container to reload config). If
+      # you add it back, pair it with --web.config.file authentication.
       - '--web.enable-remote-write-receiver'
 
   alertmanager:
     image: prom/alertmanager:v0.28.1
     container_name: <%= app_name %>-alertmanager
     ports:
-      - "9093:9093"
+      - "127.0.0.1:9093:9093"
     volumes:
       - ./config/alertmanager/development/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
       - alertmanager_data:/alertmanager
@@ -25,14 +34,6 @@ services:
       - '--config.file=/etc/alertmanager/alertmanager.yml'
       - '--storage.path=/alertmanager'
 
-  playwright:
-    image: jacoblincool/playwright:chromium-server-1.56.1
-    container_name: <%= app_name %>-playwright
-    ports:
-      - "53333:53333"
-    environment:
-      - DEBUG=true
-
 volumes:
   prometheus_data:
   alertmanager_data:

data/lib/generators/upright/install/templates/omniauth.rb

@@ -1,8 +1,12 @@
-# WARNING: Change the default password before deploying to production!
-# Set the ADMIN_PASSWORD environment variable or update the credentials below.
+# Set ADMIN_PASSWORD to enable the built-in admin login. When it is unset the
+# static credential is left unconfigured (fail closed) so the app never ships a
+# well-known default password. Change the username/credentials below to suit, or
+# switch to an external provider (e.g. OpenID Connect) via config.auth_provider.
+
+admin_password = ENV["ADMIN_PASSWORD"].presence
 
 Rails.application.config.middleware.use OmniAuth::Builder do
   provider :static_credentials,
     title: "Sign In",
-    credentials: { "admin" => ENV.fetch("ADMIN_PASSWORD", "upright") }
+    credentials: admin_password ? { "admin" => admin_password } : {}
 end

data/lib/generators/upright/install/templates/otel_collector.yml

@@ -2,6 +2,12 @@ receivers:
   otlp:
     protocols:
       http:
+        # The app container reaches the collector over the private Docker
+        # network, so the receiver must listen on a non-loopback interface.
+        # Kamal does not publish this port to the host (no `port:` on the
+        # accessory), so it is only reachable from containers on that network.
+        # If you ever publish it, bind a specific internal interface instead
+        # of 0.0.0.0 and require authentication.
         endpoint: 0.0.0.0:4318
 
 processors:

data/lib/generators/upright/install/templates/recurring.yml

@@ -23,3 +23,23 @@ production:
   cleanup_stale_probe_results:
     command: "Upright::ProbeResult.cleanup_stale"
     schedule: every hour at minute 30
+
+  sweep_playwright_videos:
+    class: "Upright::PlaywrightVideoSweepJob"
+    schedule: every hour at minute 45
+
+  health_metrics:
+    class: "Upright::HealthMetricsJob"
+    schedule: every minute
+
+  aggregate_rollups:
+    class: "Upright::Rollups::DailyAggregationJob"
+    schedule: every hour at minute 5
+
+  advance_maintenances:
+    class: "Upright::MaintenanceAdvanceJob"
+    schedule: "*/15 * * * * *"
+
+  report_incidents:
+    class: "Upright::IncidentReporterJob"
+    schedule: "*/30 * * * * *"

data/lib/generators/upright/install/templates/sites.yml

@@ -8,6 +8,8 @@ shared:
       city: London
       country: GB
       geohash: gcpvj0
+      stores_metrics: true
+      primary: true
 
 # Multi-site example:
 #
@@ -18,6 +20,8 @@ shared:
 #       country: NL
 #       geohash: u17982
 #       provider: digitalocean
+#       stores_metrics: true
+#       primary: true
 #
 #     - code: nyc
 #       city: New York City

data/lib/generators/upright/install/templates/upright.rb

@@ -5,8 +5,27 @@ Upright.configure do |config|
   config.user_agent   = "<%= Rails.application.class.module_parent_name.underscore %>/1.0"
   config.hostname     = Rails.env.local? ? "<%= Rails.application.class.module_parent_name.underscore.dasherize %>.localhost" : "<%= Rails.application.class.module_parent_name.underscore.dasherize %>.com"
 
-  # Playwright browser server URL
-  # config.playwright_server_url = ENV["PLAYWRIGHT_SERVER_URL"]
+  # Playwright CLI path (defaults to "npx playwright", override with PLAYWRIGHT_CLI_PATH env var)
+  # config.playwright_cli_path = "npx playwright"
+
+  # Token authenticating machine callers: collectors writing metrics and jobs
+  # reading a peer site's /prometheus and /alertmanager proxies. Randomly
+  # generated at install time — every site must share the same value, so treat
+  # it like any other secret when adding sites. Set the PROMETHEUS_OTLP_TOKEN
+  # env var (already wired as a Kamal secret in config/deploy.yml) to rotate it
+  # without editing this file.
+  config.proxy_token = ENV.fetch("PROMETHEUS_OTLP_TOKEN", "<%= SecureRandom.hex(32) %>")
+
+  # Viewer that trace artifacts link to, https://trace.playwright.dev by
+  # default. Upright doesn't serve one: the viewer renders a trace's contents as
+  # HTML in its own origin, so a viewer on this hostname would let a probe
+  # artifact run script against the admin session, and Upright refuses a URL
+  # under config.hostname for that reason.
+  #
+  # Following a link hands the viewer's origin a URL it can read for 24 hours.
+  # Point this at a viewer you host to keep traces to yourself, or assign nil to
+  # keep them download-only, for `npx playwright show-trace`.
+  # config.trace_viewer_url = "https://traces.example.net/index.html"
 
   # OpenTelemetry endpoint
   # config.otel_endpoint = ENV["OTEL_EXPORTER_OTLP_ENDPOINT"]

data/lib/generators/upright/install/templates/upright.rules.yml

@@ -6,20 +6,27 @@ groups:
     interval: 30s
     rules:
       # Fraction of regions reporting DOWN (0.0 to 1.0)
+      # Keep probe_service so the public status page can read live status per service.
       - record: upright:probe_down_fraction
         expr: |
-          count by (name, type, probe_target, alert_severity) (upright_probe_up == 0)
+          (
+            count by (name, type, probe_target, alert_severity, probe_service) (upright_probe_up == 0)
+            or
+            count by (name, type, probe_target, alert_severity, probe_service) (upright_probe_up) * 0
+          )
           /
-          count by (name, type, probe_target, alert_severity) (upright_probe_up)
+          count by (name, type, probe_target, alert_severity, probe_service) (upright_probe_up)
 
       # Daily uptime percentage (0.0 to 1.0)
       # Uptime = percentage of time in past day when majority of sites reported UP
+      # Keep probe_service in the grouping so ProbeRollup can tie rollups back
+      # to their Service; seed-prometheus emits the same label set.
       - record: upright:probe_uptime_daily
         expr: |
           avg_over_time((
-            sum by (name, type, probe_target) (upright_probe_up)
+            sum by (name, type, probe_target, alert_severity, probe_service) (upright_probe_up)
             /
-            count by (name, type, probe_target) (upright_probe_up)
+            count by (name, type, probe_target, alert_severity, probe_service) (upright_probe_up)
             > bool 0.5
           )[1d:])
 

data/lib/upright/configuration.rb

@@ -2,6 +2,11 @@ class Upright::Configuration
   # Global subdomain is always "app" - this is documented behavior
   GLOBAL_SUBDOMAIN = "app"
 
+  # Public status pages live at status.<hostname>; custom CNAMEs match by subdomain.
+  PUBLIC_STATUS_SUBDOMAIN = "status"
+
+  DEFAULT_TRACE_VIEWER_URL = "https://trace.playwright.dev/"
+
   # Core settings
   attr_accessor :service_name
   attr_accessor :user_agent
@@ -10,6 +15,7 @@ class Upright::Configuration
   # Storage paths
   attr_accessor :prometheus_dir
   attr_accessor :video_storage_dir
+  attr_accessor :recording_base_dir
   attr_accessor :storage_state_dir
   attr_accessor :frozen_record_path
 
@@ -17,8 +23,13 @@ class Upright::Configuration
   attr_writer :probes_path
   attr_writer :authenticators_path
 
+  # Public status page services definition (env-overridable, like probes_path)
+  attr_writer :services_path
+
   # Playwright
-  attr_accessor :playwright_server_url
+  attr_accessor :playwright_cli_path
+
+  attr_reader :trace_viewer_url
 
   # Authentication
   attr_accessor :auth_provider
@@ -29,6 +40,11 @@ class Upright::Configuration
   attr_accessor :prometheus_url
   attr_accessor :alert_webhook_url
 
+  attr_writer :proxy_token
+
+  attr_writer :rollup_minimum_coverage
+  attr_writer :rollup_evaluation_interval
+
   # Probe types
   attr_reader :probe_types
 
@@ -37,6 +53,15 @@ class Upright::Configuration
   attr_accessor :stale_failure_threshold
   attr_accessor :failure_retention_limit
 
+  # Public status pages
+  attr_accessor :public_status_enabled
+  attr_reader :public_status_custom_domains
+
+  # Extra stylesheets host apps layer on top of the engine's for the public
+  # status page (theming, branding). Logical asset names, loaded last so their
+  # :root overrides win the cascade.
+  attr_writer :public_stylesheets
+
   def initialize
     @service_name = "upright"
     @user_agent = "Upright/1.0"
@@ -44,15 +69,18 @@ class Upright::Configuration
 
     @prometheus_dir = nil
     @video_storage_dir = nil
+    @recording_base_dir = nil
     @storage_state_dir = nil
     @frozen_record_path = nil
     @probes_path = nil
     @authenticators_path = nil
+    @services_path = nil
 
     @probe_types = Upright::ProbeTypeRegistry.new
 
-    @playwright_server_url = ENV["PLAYWRIGHT_SERVER_URL"]
+    @playwright_cli_path = ENV.fetch("PLAYWRIGHT_CLI_PATH", "npx playwright")
     @otel_endpoint = ENV["OTEL_EXPORTER_OTLP_ENDPOINT"]
+    @prometheus_url = nil
 
     @auth_provider = :static_credentials
     @auth_options = {}
@@ -60,6 +88,40 @@ class Upright::Configuration
     @stale_success_threshold = 24.hours
     @stale_failure_threshold = 30.days
     @failure_retention_limit = 20_000
+
+    self.trace_viewer_url = ENV.fetch("TRACE_VIEWER_URL", DEFAULT_TRACE_VIEWER_URL)
+
+    @public_status_enabled = false
+    @public_status_custom_domains = []
+    @public_stylesheets = nil
+  end
+
+  def trace_viewer_url=(url)
+    @trace_viewer_url = url.presence
+    host = trace_viewer_uri&.host
+
+    if @trace_viewer_url && (host.nil? || admin_domain?(host))
+      raise Upright::ConfigurationError, "config.trace_viewer_url must be an http(s) URL outside #{@hostname}"
+    end
+  end
+
+  def trace_viewer_origin
+    trace_viewer_uri&.then do |url|
+      "#{url.scheme}://#{url.host}#{":#{url.port}" unless url.port == url.default_port}"
+    end
+  end
+
+  def public_status_subdomain
+    PUBLIC_STATUS_SUBDOMAIN
+  end
+
+  def public_stylesheets
+    Array(@public_stylesheets)
+  end
+
+  def public_status_custom_domains=(domains)
+    @public_status_custom_domains = Array(domains)
+    configure_allowed_hosts if @hostname
   end
 
   def global_subdomain
@@ -74,10 +136,30 @@ class Upright::Configuration
     @prometheus_dir || Rails.root.join("tmp", "prometheus")
   end
 
+  def prometheus_url
+    @prometheus_url || ENV.fetch("PROMETHEUS_URL", "http://localhost:9090")
+  end
+
+  def proxy_token
+    @proxy_token || ENV["PROMETHEUS_OTLP_TOKEN"]
+  end
+
+  def rollup_minimum_coverage
+    @rollup_minimum_coverage || 0.9
+  end
+
+  def rollup_evaluation_interval
+    @rollup_evaluation_interval || 30.seconds
+  end
+
   def video_storage_dir
     @video_storage_dir || Rails.root.join("storage", "playwright_videos")
   end
 
+  def recording_base_dir
+    @recording_base_dir || Rails.root.join("storage", "playwright_recordings")
+  end
+
   def storage_state_dir
     @storage_state_dir || Rails.root.join("storage", "playwright_storage_states")
   end
@@ -90,6 +172,10 @@ class Upright::Configuration
     @probes_path || Rails.root.join("probes")
   end
 
+  def services_path
+    @services_path || Rails.root.join("config", "services.yml")
+  end
+
   def authenticators_path
     @authenticators_path || Rails.root.join("probes", "authenticators")
   end
@@ -104,17 +190,38 @@ class Upright::Configuration
   end
 
   def default_url_options
-    if Rails.env.production?
-      { protocol: "https", host: "#{global_subdomain}.#{hostname}", domain: hostname }
-    else
+    if Rails.env.local?
       { protocol: "http", host: "#{global_subdomain}.#{hostname}", port: ENV.fetch("PORT", 3000).to_i, domain: hostname }
+    else
+      { protocol: "https", host: "#{global_subdomain}.#{hostname}", domain: hostname }
     end
   end
 
   private
+    def trace_viewer_uri
+      url = URI(@trace_viewer_url.to_s)
+      url if url.is_a?(URI::HTTP) && url.host.present?
+    rescue URI::InvalidURIError
+      nil
+    end
+
+    # DNS is case-insensitive and tolerates a trailing dot, so compare folded.
+    def admin_domain?(host)
+      return false if @hostname.blank?
+
+      host = host.downcase.chomp(".")
+      admin = @hostname.downcase
+
+      host == admin || host.end_with?(".#{admin}")
+    end
+
     def configure_allowed_hosts
       port_suffix = Rails.env.local? ? "(:\\d+)?" : ""
-      Rails.application.config.hosts = [ /.*\.#{Regexp.escape(hostname)}#{port_suffix}/, /#{Regexp.escape(hostname)}#{port_suffix}/ ]
+      hosts = [ /.*\.#{Regexp.escape(hostname)}#{port_suffix}/, /#{Regexp.escape(hostname)}#{port_suffix}/ ]
+      Array(@public_status_custom_domains).each do |domain|
+        hosts << /\A#{Regexp.escape(domain)}#{port_suffix}\z/
+      end
+      Rails.application.config.hosts = hosts
       Rails.application.config.action_dispatch.tld_length = 1
     end
 end

data/lib/upright/engine.rb

@@ -6,14 +6,34 @@ class Upright::Engine < ::Rails::Engine
 
   # Session store configuration
   initializer "upright.session_store", before: :load_config_initializers do |app|
+    # Scope the cookie to the configured hostname (resolved per-request, since the
+    # host app sets it in an initializer that runs after this one) rather than
+    # `domain: :all`, which — with a multi-label hostname like upright.example.com
+    # — would hand the cookie to the registrable parent and every sibling under it
+    # (F-08). This is independent of tld_length, so custom status domains still
+    # parse normally.
     app.config.session_store :cookie_store,
       key: "_upright_session",
-      domain: :all,
+      domain: ->(_request) { Upright.configuration.hostname&.then { |h| ".#{h}" } },
       same_site: :lax,
-      secure: Rails.env.production?,
+      secure: !Rails.env.local?,
+      httponly: true,
       expire_after: 24.hours
   end
 
+  # The proxy token stands in for an admin session on the metrics proxies, so a
+  # blank one in a deployed environment is a fail-open: any bearer value would be
+  # compared against "" . Refuse to boot without one outside local envs — but not
+  # during `assets:precompile`, which Rails runs with SECRET_KEY_BASE_DUMMY and no
+  # runtime secrets (the Docker image builds before the token is available).
+  config.after_initialize do
+    building_assets = ENV["SECRET_KEY_BASE_DUMMY"].present?
+
+    if !Rails.env.local? && !building_assets && Upright.configuration.proxy_token.blank?
+      raise Upright::ConfigurationError, "Upright.configuration.proxy_token (or PROMETHEUS_OTLP_TOKEN) must be set outside development/test"
+    end
+  end
+
   config.after_initialize do
     url_options = Upright.configuration.default_url_options
     Rails.application.routes.default_url_options = url_options
@@ -125,6 +145,9 @@ class Upright::Engine < ::Rails::Engine
       Upright.sites.each do |site|
         puts "  #{site.city || site.code}:  #{protocol}://#{site.code}.#{hostname}:#{port}"
       end
+      if Upright.configuration.public_status_enabled
+        puts "  Public status: #{protocol}://#{Upright.configuration.public_status_subdomain}.#{hostname}:#{port}"
+      end
       puts ""
     end
   end

data/lib/upright/metrics.rb

@@ -55,6 +55,31 @@ module Upright::Metrics
               comment: "HTTP response status code",
               aggregation: :max,
               tags: %i[name probe_target probe_service]
+
+            gauge :service_under_maintenance,
+              comment: "Whether a service is under active maintenance (1 = yes, 0 = no)",
+              aggregation: :most_recent,
+              tags: %i[probe_service]
+
+            gauge :primary_site,
+              comment: "Whether this site is the primary (1 = yes, 0 = no)",
+              aggregation: :most_recent,
+              tags: []
+
+            gauge :persistent_db_up,
+              comment: "Whether the persistent database answers (1 = yes, 0 = no)",
+              aggregation: :most_recent,
+              tags: []
+
+            gauge :rollup_skipped_probes,
+              comment: "Probes the last rollup run left unwritten for insufficient Prometheus coverage",
+              aggregation: :most_recent,
+              tags: []
+
+            gauge :rollup_last_run_timestamp_seconds,
+              comment: "Unix time of the most recently written daily rollup",
+              aggregation: :max,
+              tags: []
           end
         end
       end

data/lib/upright/site.rb

@@ -2,19 +2,37 @@ module Upright
   class Site
     attr_reader :code, :city, :country, :geohash, :stagger_index
 
-    def initialize(code:, city: nil, country: nil, geohash: nil, provider: nil, stagger_index: 0)
+    def initialize(code:, city: nil, country: nil, geohash: nil, provider: nil, stores_metrics: false, primary: false, stagger_index: 0)
       @code = code.to_sym
       @city = city
       @country = country
       @geohash = geohash
       @provider = provider
+      @stores_metrics = stores_metrics
+      @primary = primary
       @stagger_index = stagger_index
     end
 
+    def stores_metrics?
+      @stores_metrics
+    end
+
+    def primary?
+      @primary
+    end
+
     def host
       URI.parse(url).host
     end
 
+    def current?
+      code == Upright.current_site&.code
+    end
+
+    def prometheus_client
+      current? ? Upright.prometheus_client : peer_prometheus_client
+    end
+
     def provider
       @provider.to_s.inquiry
     end
@@ -40,6 +58,18 @@ module Upright
     end
 
     private
+      def peer_prometheus_client
+        Prometheus::ApiClient.client(url: prometheus_proxy_url, headers: proxy_authorization, options: { timeout: 30 })
+      end
+
+      def prometheus_proxy_url
+        "#{url.chomp("/")}/prometheus"
+      end
+
+      def proxy_authorization
+        { "Authorization" => "Bearer #{Upright.configuration.proxy_token}" }
+      end
+
       def coordinates
         @coordinates ||= Upright::Geohash.decode(geohash).first
       end

data/lib/upright/version.rb

@@ -1,3 +1,5 @@
 module Upright
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
+  # Keep in sync with the playwright-ruby-client constraint in upright.gemspec
+  PLAYWRIGHT_VERSION = "1.59"
 end

data/lib/upright.rb

@@ -13,6 +13,7 @@ require "propshaft"
 require "importmap-rails"
 require "turbo-rails"
 require "stimulus-rails"
+require "local_time"
 require "geared_pagination"
 require "yabeda/prometheus"
 require "yabeda/puma/plugin"
@@ -43,6 +44,13 @@ module Upright
       configuration.probe_types
     end
 
+    def prometheus_client
+      Prometheus::ApiClient.client(
+        url: configuration.prometheus_url,
+        options: { timeout: 30.seconds }
+      )
+    end
+
     def sites
       @sites ||= load_sites
     end
@@ -52,7 +60,15 @@ module Upright
     end
 
     def current_site
-      find_site(ENV["SITE_SUBDOMAIN"]) || sites.first
+      if (subdomain = ENV["SITE_SUBDOMAIN"]).present?
+        find_site(subdomain) || raise(ConfigurationError, "SITE_SUBDOMAIN=#{subdomain} is not a site in sites.yml (#{sites.map(&:code).join(', ')})")
+      else
+        sites.first
+      end
+    end
+
+    def primary_site
+      sites.find(&:primary?)
     end
 
     private
@@ -64,10 +80,21 @@ module Upright
 
           config[:sites].map.with_index do |site_config, index|
             Site.new(stagger_index: index, **site_config)
-          end
+          end.tap { |sites| ensure_at_most_one_primary(sites) }
         else
           []
         end
       end
+
+      # Two primaries means two hosts running the singleton jobs that write the
+      # shared persistent database. None is how a host looks before it adopts
+      # the flag, so that stays legal.
+      def ensure_at_most_one_primary(sites)
+        primaries = sites.select(&:primary?)
+
+        if primaries.many?
+          raise ConfigurationError, "sites.yml declares #{primaries.count} primary sites (#{primaries.map(&:code).join(', ')}); at most one may be primary"
+        end
+      end
   end
 end