upright 0.2.0 → 0.4.0

data/lib/generators/upright/install/database_config.rb

@@ -0,0 +1,46 @@
+module Upright
+  module Generators
+    # Rewrites the single-database development and test entries that `rails new`
+    # writes into the three databases Upright uses:
+    #
+    #   primary     probe results and Active Storage artifacts
+    #   persistent  rollups, incidents and maintenance windows, which outlive a
+    #               site's probe data; its migrations ship with the gem
+    #   queue       Solid Queue
+    #
+    # Pure string transformation so the install generator's edit can be tested
+    # without a generated application.
+    module DatabaseConfig
+      ENVIRONMENTS = %w[development test].freeze
+
+      PERSISTENT_MIGRATIONS = %(<%= Gem.loaded_specs["upright"].gem_dir %>/db/persistent_migrate)
+
+      def self.rewrite(yaml)
+        ENVIRONMENTS.reduce(yaml) do |rewritten, env|
+          rewritten.sub(single_database(env), split_databases(env))
+        end
+      end
+
+      def self.single_database(env)
+        "#{env}:\n  <<: *default\n  database: storage/#{env}.sqlite3"
+      end
+
+      def self.split_databases(env)
+        <<~YAML.chomp
+          #{env}:
+            primary:
+              <<: *default
+              database: storage/#{env}.sqlite3
+            persistent:
+              <<: *default
+              database: storage/#{env}_persistent.sqlite3
+              migrations_paths: #{PERSISTENT_MIGRATIONS}
+            queue:
+              <<: *default
+              database: storage/#{env}_queue.sqlite3
+              migrations_paths: db/queue_migrate
+        YAML
+      end
+    end
+  end
+end

data/lib/generators/upright/install/install_generator.rb

@@ -1,3 +1,5 @@
+require_relative "database_config"
+
 module Upright
   module Generators
     class InstallGenerator < Rails::Generators::Base
@@ -5,9 +7,23 @@ module Upright
 
       desc "Install Upright engine into your application"
 
+      def set_ruby_version
+        ruby_version = RUBY_VERSION
+        create_file ".ruby-version", "#{ruby_version}\n"
+        create_file "mise.toml", <<~TOML
+          [tools]
+          ruby = "#{ruby_version}"
+        TOML
+      end
+
       def copy_initializers
         template "upright.rb", "config/initializers/upright.rb"
         template "omniauth.rb", "config/initializers/omniauth.rb"
+        # Never force: an app may already run an enforced CSP, and silently
+        # replacing it with this commented template would disable it. On a
+        # collision Thor prompts instead of overwriting; the recommended policy is
+        # documented for the operator to merge.
+        template "content_security_policy.rb", "config/initializers/content_security_policy.rb"
       end
 
       def copy_sites_config
@@ -37,12 +53,45 @@ module Upright
 
       def copy_deploy_config
         template "deploy.yml", "config/deploy.yml"
+        template "Dockerfile", "Dockerfile"
       end
 
       def copy_puma_config
         template "puma.rb", "config/puma.rb"
       end
 
+      def install_solid_queue
+        rails_command "solid_queue:install"
+      end
+
+      # Splits the development and test databases into primary, persistent and
+      # queue. The persistent database holds rollups, incidents and maintenance
+      # windows, and its migrations come from the gem, so database.yml points
+      # migrations_paths at the installed gem's db/persistent_migrate.
+      def add_persistent_and_queue_databases
+        gsub_file "config/database.yml", /\A.*\z/m do |yaml|
+          DatabaseConfig.rewrite(yaml)
+        end
+      end
+
+      def copy_recurring_config
+        template "recurring.yml", "config/recurring.yml", force: true
+      end
+
+      def add_jobs_to_procfile
+        procfile = File.join(destination_root, "Procfile.dev")
+        if File.exist?(procfile)
+          unless File.read(procfile).include?("jobs:")
+            append_to_file "Procfile.dev", "jobs: OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES bin/rails solid_queue:start\n"
+          end
+        else
+          create_file "Procfile.dev", <<~PROCFILE
+            web: bin/rails server -b '0.0.0.0' -p ${PORT:-3000}
+            jobs: OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES bin/rails solid_queue:start
+          PROCFILE
+        end
+      end
+
       def add_routes
         route 'mount Upright::Engine => "/", as: :upright'
       end
@@ -60,14 +109,14 @@ module Upright
         say "Upright has been installed!", :green
         say ""
         say "Next steps:"
-        say "  1. Run migrations: bin/rails db:migrate"
+        say "  1. Prepare the databases (primary, persistent and queue): bin/rails db:prepare"
         say "  2. Configure your servers in config/deploy.yml"
         say "  3. Configure sites in config/sites.yml"
         say "  4. Add probes in probes/*.yml"
-        say "  5. Set ADMIN_PASSWORD env var (default: upright)"
+        say "  5. Set the ADMIN_PASSWORD env var — required, there is no default password"
         say ""
         say "For production, review config/initializers/upright.rb and update:"
-        say "  config.hostname = \"honcho-upright.com\""
+        say "  config.hostname = \"example.com\""
         say ""
         say "Start dev services (Prometheus, Alertmanager, Playwright):"
         say "  docker compose up -d"

data/lib/generators/upright/install/templates/Dockerfile

@@ -0,0 +1,80 @@
+# syntax = docker/dockerfile:1
+
+# This Dockerfile is designed for production, not development. Use with Kamal or build'n'run by hand:
+# docker build -t my-app .
+# docker run -d -p 80:80 -p 443:443 --name my-app -e RAILS_MASTER_KEY=<value from config/master.key> my-app
+
+# Make sure RUBY_VERSION matches the Ruby version in .ruby-version
+ARG RUBY_VERSION=<%= RUBY_VERSION %>
+FROM docker.io/library/ruby:$RUBY_VERSION-slim AS base
+
+# Rails app lives here
+WORKDIR /rails
+
+# Install base packages and Node.js (for Playwright)
+RUN apt-get update -qq && \
+    apt-get install --no-install-recommends -y curl libjemalloc2 libvips sqlite3 iputils-ping mtr-tiny \
+    nodejs npm && \
+    rm -rf /var/lib/apt/lists /var/cache/apt/archives
+
+# Set production environment
+ENV RAILS_ENV="production" \
+    BUNDLE_DEPLOYMENT="1" \
+    BUNDLE_PATH="/usr/local/bundle" \
+    BUNDLE_WITHOUT="development"
+
+# Throw-away build stage to reduce size of final image
+FROM base AS build
+
+# Install packages needed to build gems
+RUN apt-get update -qq && \
+    apt-get install --no-install-recommends -y build-essential git pkg-config libssl-dev libyaml-dev rustc cargo libclang-dev && \
+    rm -rf /var/lib/apt/lists /var/cache/apt/archives
+
+# Install application gems
+COPY Gemfile Gemfile.lock ./
+RUN --mount=type=secret,id=GITHUB_TOKEN \
+    BUNDLE_GITHUB__COM="$(cat /run/secrets/GITHUB_TOKEN):x-oauth-basic" bundle install && \
+    rm -rf ~/.bundle/ "${BUNDLE_PATH}"/ruby/*/cache "${BUNDLE_PATH}"/ruby/*/bundler/gems/*/.git && \
+    bundle exec bootsnap precompile --gemfile && \
+    bundle exec ruby -e "require 'upright/version'; puts Upright::PLAYWRIGHT_VERSION" > /tmp/playwright_version
+
+# Copy application code
+COPY . .
+
+# Precompile bootsnap code for faster boot times
+RUN bundle exec bootsnap precompile app.rb probes/
+
+# Precompiling assets for production without requiring secret RAILS_MASTER_KEY
+RUN SECRET_KEY_BASE_DUMMY=1 ./bin/rails assets:precompile
+
+# Final stage for app image
+FROM base
+
+# Install Playwright before copying app code so the layer is cached on code-only deploys.
+# The version is extracted to a file in the build stage; BuildKit's content-based caching
+# keeps this layer cached even across Gemfile changes when the version is unchanged.
+COPY --from=build /tmp/playwright_version /tmp/playwright_version
+ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
+RUN PLAYWRIGHT_VERSION=$(cat /tmp/playwright_version) && \
+    npm install -g playwright@~${PLAYWRIGHT_VERSION}.0 && \
+    playwright install --with-deps chromium && \
+    chmod -R o+rx /ms-playwright
+
+# Copy built artifacts: gems, application
+COPY --from=build "${BUNDLE_PATH}" "${BUNDLE_PATH}"
+COPY --from=build /rails /rails
+
+# Run and own only the runtime files as a non-root user for security
+RUN groupadd --system --gid 1000 rails && \
+    useradd rails --uid 1000 --gid 1000 --create-home --shell /bin/bash && \
+    mkdir -p storage tmp && \
+    chown -R rails:rails storage tmp
+USER 1000:1000
+
+# Entrypoint prepares the database.
+ENTRYPOINT ["/rails/bin/docker-entrypoint"]
+
+# Start the server by default, this can be overwritten at runtime
+EXPOSE 3000 9394
+CMD ["./bin/rails", "server"]

data/lib/generators/upright/install/templates/content_security_policy.rb

@@ -0,0 +1,42 @@
+# Be sure to restart your server when you modify this file.
+
+# Content Security Policy tuned for Upright. Uncomment the block below to
+# enforce it. Every source the engine actually uses is listed explicitly:
+#
+#   * JavaScript dependencies (Turbo, Stimulus, Leaflet, Frappe Charts) are
+#     pinned to https://cdn.jsdelivr.net in the engine's config/importmap.rb.
+#   * The Leaflet stylesheet is loaded from https://unpkg.com in the engine
+#     layout; Turbo injects an inline <style> for its progress bar, which is
+#     why style-src keeps :unsafe_inline (styles only — scripts stay strict).
+#   * Map tiles come from OpenStreetMap (light) and Carto (dark); Leaflet's
+#     default marker icons load from unpkg alongside its stylesheet.
+#   * The inline <script type="importmap"> tag is authorized by a per-session
+#     nonce via csp_meta_tag, which the engine layouts already render.
+#
+# To verify without breaking anything, start with
+# config.content_security_policy_report_only = true and watch the browser
+# console before enforcing.
+
+# Rails.application.configure do
+#   config.content_security_policy do |policy|
+#     policy.default_src     :self
+#     policy.font_src        :self, :data
+#     policy.img_src         :self, :data, "https://unpkg.com",
+#                            "https://*.tile.openstreetmap.org"
+#     policy.object_src      :none
+#     policy.script_src      :self, "https://cdn.jsdelivr.net"
+#     policy.style_src       :self, :unsafe_inline, "https://unpkg.com"
+#     policy.connect_src     :self
+#     policy.frame_ancestors :self
+#     policy.base_uri        :self
+#     # Specify URI for violation reports
+#     # policy.report_uri "/csp-violation-report-endpoint"
+#   end
+#
+#   # Generate session nonces for the inline importmap script tag.
+#   config.content_security_policy_nonce_generator = ->(request) { request.session.id.to_s }
+#   config.content_security_policy_nonce_directives = %w(script-src)
+#
+#   # Report violations without enforcing the policy.
+#   # config.content_security_policy_report_only = true
+# end

data/lib/generators/upright/install/templates/deploy.yml

@@ -48,6 +48,7 @@ env:
   secret:
     - RAILS_MASTER_KEY
     - PROMETHEUS_OTLP_TOKEN
+    - ADMIN_PASSWORD
   tags:
     london:
       SITE_SUBDOMAIN: lon
@@ -62,7 +63,13 @@ volumes:
 
 accessories:
   otel_collector:
-    image: otel/opentelemetry-collector-contrib:0.126.0
+    # Pinned by digest so deploys are immutable; the tag is documentation only.
+    image: otel/opentelemetry-collector-contrib:0.126.0@sha256:973747f78266a8ffec428417727e6b704559e9a30e4be8c1cca2899facd68689
+    # The collector runs unprivileged and only receives OTLP over the private
+    # Docker network — it does not mount the Docker socket or host log
+    # directories. To collect container logs, prefer a filelog receiver over
+    # a read-only mount of the specific log path rather than the Docker socket,
+    # which grants root-equivalent control of the host.
     env:
       secret:
         - OTEL_GATEWAY_USERNAME
@@ -71,17 +78,6 @@ accessories:
         - PROMETHEUS_OTLP_TOKEN
     files:
       - config/otel_collector.yml:/etc/otelcol-contrib/config.yaml
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-      - /var/lib/docker/containers:/var/lib/docker/containers
-    options:
-      user: 0
-    roles:
-      - jobs
-
-  playwright:
-    image: jacoblincool/playwright:chromium-server-1.55.0
-    port: "127.0.0.1:53333:53333"
     roles:
       - jobs
 
@@ -89,12 +85,16 @@ accessories:
     image: prom/prometheus:v3.2.1
     hosts:
       - <%= app_domain %>
+    # No published port: Prometheus is only reachable over the private Docker
+    # network (the app proxies it). --web.enable-lifecycle is deliberately
+    # omitted — the unauthenticated reload/quit endpoints aren't needed since
+    # config changes redeploy the accessory; if you add it back, front it with
+    # --web.config.file authentication.
     cmd: >-
       --config.file=/etc/prometheus/prometheus.yml
       --storage.tsdb.path=/prometheus
       --storage.tsdb.retention.time=30d
       --web.enable-otlp-receiver
-      --web.enable-lifecycle
     files:
       - config/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
       - config/prometheus/rules/upright.rules.yml:/etc/prometheus/rules/upright.rules.yml

data/lib/generators/upright/install/templates/development_prometheus.yml

@@ -9,4 +9,4 @@ scrape_configs:
   - job_name: "upright"
     scrape_interval: 5s
     static_configs:
-      - targets: ["localhost:9394"]
+      - targets: ["host.docker.internal:9394"]

data/lib/generators/upright/install/templates/docker-compose.yml

@@ -2,7 +2,14 @@ services:
   prometheus:
     image: prom/prometheus:v3.4.1
     container_name: <%= app_name %>-prometheus
-    network_mode: host
+    # Published on loopback only, so the dev server is not reachable from other
+    # machines. host.docker.internal lets Prometheus scrape the app running on
+    # the host, on Linux as well as on macOS and Windows Docker Desktop, where
+    # network_mode: host does not reach the host's localhost.
+    ports:
+      - "127.0.0.1:9090:9090"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     volumes:
       - ./config/prometheus/development/prometheus.yml:/etc/prometheus/prometheus.yml:ro
       - ./config/prometheus/rules:/etc/prometheus/rules:ro
@@ -10,14 +17,16 @@ services:
     command:
       - '--config.file=/etc/prometheus/prometheus.yml'
       - '--storage.tsdb.path=/prometheus'
-      - '--web.enable-lifecycle'
+      # --web.enable-lifecycle is omitted: its unauthenticated reload and quit
+      # endpoints are not needed (restart the container to reload config). If
+      # you add it back, pair it with --web.config.file authentication.
       - '--web.enable-remote-write-receiver'
 
   alertmanager:
     image: prom/alertmanager:v0.28.1
     container_name: <%= app_name %>-alertmanager
     ports:
-      - "9093:9093"
+      - "127.0.0.1:9093:9093"
     volumes:
       - ./config/alertmanager/development/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
       - alertmanager_data:/alertmanager
@@ -25,14 +34,6 @@ services:
       - '--config.file=/etc/alertmanager/alertmanager.yml'
       - '--storage.path=/alertmanager'
 
-  playwright:
-    image: jacoblincool/playwright:chromium-server-1.56.1
-    container_name: <%= app_name %>-playwright
-    ports:
-      - "53333:53333"
-    environment:
-      - DEBUG=true
-
 volumes:
   prometheus_data:
   alertmanager_data:

data/lib/generators/upright/install/templates/http_probes.yml

@@ -7,6 +7,7 @@
 #
 # Optional fields:
 #   - expected_status: HTTP status code to expect (default: 200-399)
+#   - alert_severity: Alert severity when probe fails: medium, high (default), critical
 #   - basic_auth_credentials: Key in Rails credentials for basic auth
 #   - proxy: Key in Rails credentials for proxy settings
 

data/lib/generators/upright/install/templates/omniauth.rb

@@ -1,8 +1,12 @@
-# WARNING: Change the default password before deploying to production!
-# Set the ADMIN_PASSWORD environment variable or update the credentials below.
+# Set ADMIN_PASSWORD to enable the built-in admin login. When it is unset the
+# static credential is left unconfigured (fail closed) so the app never ships a
+# well-known default password. Change the username/credentials below to suit, or
+# switch to an external provider (e.g. OpenID Connect) via config.auth_provider.
+
+admin_password = ENV["ADMIN_PASSWORD"].presence
 
 Rails.application.config.middleware.use OmniAuth::Builder do
   provider :static_credentials,
     title: "Sign In",
-    credentials: { "admin" => ENV.fetch("ADMIN_PASSWORD", "upright") }
+    credentials: admin_password ? { "admin" => admin_password } : {}
 end

data/lib/generators/upright/install/templates/otel_collector.yml

@@ -2,6 +2,12 @@ receivers:
   otlp:
     protocols:
       http:
+        # The app container reaches the collector over the private Docker
+        # network, so the receiver must listen on a non-loopback interface.
+        # Kamal does not publish this port to the host (no `port:` on the
+        # accessory), so it is only reachable from containers on that network.
+        # If you ever publish it, bind a specific internal interface instead
+        # of 0.0.0.0 and require authentication.
         endpoint: 0.0.0.0:4318
 
 processors:

data/lib/generators/upright/install/templates/recurring.yml

@@ -0,0 +1,45 @@
+development:
+  http_probes:
+    schedule: every minute
+    command: "Upright::Probes::HTTPProbe.check_and_record_all_later"
+
+  smtp_probes:
+    schedule: every minute
+    command: "Upright::Probes::SMTPProbe.check_and_record_all_later"
+
+production:
+  http_probes:
+    schedule: "*/30 * * * * *"
+    command: "Upright::Probes::HTTPProbe.check_and_record_all_later"
+
+  smtp_probes:
+    schedule: "*/30 * * * * *"
+    command: "Upright::Probes::SMTPProbe.check_and_record_all_later"
+
+  clear_solid_queue_finished_jobs:
+    command: "SolidQueue::Job.clear_finished_in_batches(sleep_between_batches: 0.3)"
+    schedule: every hour at minute 12
+
+  cleanup_stale_probe_results:
+    command: "Upright::ProbeResult.cleanup_stale"
+    schedule: every hour at minute 30
+
+  sweep_playwright_videos:
+    class: "Upright::PlaywrightVideoSweepJob"
+    schedule: every hour at minute 45
+
+  health_metrics:
+    class: "Upright::HealthMetricsJob"
+    schedule: every minute
+
+  aggregate_rollups:
+    class: "Upright::Rollups::DailyAggregationJob"
+    schedule: every hour at minute 5
+
+  advance_maintenances:
+    class: "Upright::MaintenanceAdvanceJob"
+    schedule: "*/15 * * * * *"
+
+  report_incidents:
+    class: "Upright::IncidentReporterJob"
+    schedule: "*/30 * * * * *"

data/lib/generators/upright/install/templates/sites.yml

@@ -8,6 +8,8 @@ shared:
       city: London
       country: GB
       geohash: gcpvj0
+      stores_metrics: true
+      primary: true
 
 # Multi-site example:
 #
@@ -18,6 +20,8 @@ shared:
 #       country: NL
 #       geohash: u17982
 #       provider: digitalocean
+#       stores_metrics: true
+#       primary: true
 #
 #     - code: nyc
 #       city: New York City

data/lib/generators/upright/install/templates/smtp_probes.yml

@@ -4,6 +4,9 @@
 # Required fields:
 #   - name: Unique identifier for the probe
 #   - host: SMTP server hostname
+#
+# Optional fields:
+#   - alert_severity: Alert severity when probe fails: medium, high (default), critical
 
 # - name: Example Mail Server
 #   host: mail.example.com

data/lib/generators/upright/install/templates/traceroute_probes.yml

@@ -4,6 +4,9 @@
 # Required fields:
 #   - name: Unique identifier for the probe
 #   - host: Hostname or IP address to trace
+#
+# Optional fields:
+#   - alert_severity: Alert severity when probe fails: medium, high (default), critical
 
 # - name: Example Service
 #   host: example.com

data/lib/generators/upright/install/templates/upright.rb

@@ -5,8 +5,27 @@ Upright.configure do |config|
   config.user_agent   = "<%= Rails.application.class.module_parent_name.underscore %>/1.0"
   config.hostname     = Rails.env.local? ? "<%= Rails.application.class.module_parent_name.underscore.dasherize %>.localhost" : "<%= Rails.application.class.module_parent_name.underscore.dasherize %>.com"
 
-  # Playwright browser server URL
-  # config.playwright_server_url = ENV["PLAYWRIGHT_SERVER_URL"]
+  # Playwright CLI path (defaults to "npx playwright", override with PLAYWRIGHT_CLI_PATH env var)
+  # config.playwright_cli_path = "npx playwright"
+
+  # Token authenticating machine callers: collectors writing metrics and jobs
+  # reading a peer site's /prometheus and /alertmanager proxies. Randomly
+  # generated at install time — every site must share the same value, so treat
+  # it like any other secret when adding sites. Set the PROMETHEUS_OTLP_TOKEN
+  # env var (already wired as a Kamal secret in config/deploy.yml) to rotate it
+  # without editing this file.
+  config.proxy_token = ENV.fetch("PROMETHEUS_OTLP_TOKEN", "<%= SecureRandom.hex(32) %>")
+
+  # Viewer that trace artifacts link to, https://trace.playwright.dev by
+  # default. Upright doesn't serve one: the viewer renders a trace's contents as
+  # HTML in its own origin, so a viewer on this hostname would let a probe
+  # artifact run script against the admin session, and Upright refuses a URL
+  # under config.hostname for that reason.
+  #
+  # Following a link hands the viewer's origin a URL it can read for 24 hours.
+  # Point this at a viewer you host to keep traces to yourself, or assign nil to
+  # keep them download-only, for `npx playwright show-trace`.
+  # config.trace_viewer_url = "https://traces.example.net/index.html"
 
   # OpenTelemetry endpoint
   # config.otel_endpoint = ENV["OTEL_EXPORTER_OTLP_ENDPOINT"]
@@ -18,4 +37,7 @@ Upright.configure do |config|
   #   client_id: ENV["OIDC_CLIENT_ID"],
   #   client_secret: ENV["OIDC_CLIENT_SECRET"]
   # }
+
+  # Register custom probe types (built-in types: http, playwright, smtp, traceroute)
+  # config.probe_types.register :ftp_file, name: "FTP File", icon: "📂"
 end

data/lib/generators/upright/install/templates/upright.rules.yml

@@ -6,20 +6,27 @@ groups:
     interval: 30s
     rules:
       # Fraction of regions reporting DOWN (0.0 to 1.0)
+      # Keep probe_service so the public status page can read live status per service.
       - record: upright:probe_down_fraction
         expr: |
-          count by (name, type, probe_target) (upright_probe_up == 0)
+          (
+            count by (name, type, probe_target, alert_severity, probe_service) (upright_probe_up == 0)
+            or
+            count by (name, type, probe_target, alert_severity, probe_service) (upright_probe_up) * 0
+          )
           /
-          count by (name, type, probe_target) (upright_probe_up)
+          count by (name, type, probe_target, alert_severity, probe_service) (upright_probe_up)
 
       # Daily uptime percentage (0.0 to 1.0)
       # Uptime = percentage of time in past day when majority of sites reported UP
+      # Keep probe_service in the grouping so ProbeRollup can tie rollups back
+      # to their Service; seed-prometheus emits the same label set.
       - record: upright:probe_uptime_daily
         expr: |
           avg_over_time((
-            sum by (name, type, probe_target) (upright_probe_up)
+            sum by (name, type, probe_target, alert_severity, probe_service) (upright_probe_up)
             /
-            count by (name, type, probe_target) (upright_probe_up)
+            count by (name, type, probe_target, alert_severity, probe_service) (upright_probe_up)
             > bool 0.5
           )[1d:])
 
@@ -148,7 +155,7 @@ groups:
           upright: "https://app.<%= app_domain %>/?probe_type={{ $labels.type }}&status=fail&probe_name={{ $labels.name | urlquery }}"
         expr: upright:probe_down_fraction{type="http"} > 0.5
         labels:
-          severity: page
+          severity: "{{ $labels.alert_severity }}"
           group: upright
 
       - alert: UprightHTTPProbeDegraded
@@ -187,7 +194,7 @@ groups:
           upright: "https://app.<%= app_domain %>/?probe_type={{ $labels.type }}&status=fail&probe_name={{ $labels.name | urlquery }}"
         expr: upright:probe_down_fraction{type="smtp"} > 0.5
         labels:
-          severity: page
+          severity: "{{ $labels.alert_severity }}"
           group: upright
 
       - alert: UprightSMTPProbeDegraded
@@ -226,7 +233,7 @@ groups:
           upright: "https://app.<%= app_domain %>/?probe_type={{ $labels.type }}&status=fail&probe_name={{ $labels.name | urlquery }}"
         expr: upright:probe_down_fraction{type="playwright"} > 0.5
         labels:
-          severity: page
+          severity: "{{ $labels.alert_severity }}"
           group: upright
 
       - alert: UprightPlaywrightProbeDegraded