upright 0.2.0 → 0.4.0

data/app/models/upright/incident.rb

@@ -0,0 +1,87 @@
+class Upright::Incident < Upright::PersistentRecord
+  include Upright::Incidents::Lifecycle
+  include Upright::Incidents::AutoReporting
+
+  attr_accessor :body
+
+  enum :status, %w[ investigating monitoring resolved
+                    scheduled in_progress completed ].index_with(&:itself)
+  enum :impact, %w[ minor major critical maintenance ].index_with(&:itself), prefix: true, default: "minor"
+
+  STATUSES          = %w[ investigating monitoring resolved ]
+  TERMINAL_STATUSES = %w[ resolved ]
+  IMPACTS           = %w[ minor major critical ]
+
+  IMPACT_STATUS = { "minor" => :degraded, "major" => :partial_outage, "critical" => :major_outage }
+
+  TEMPLATES = {
+    down: "%{services} %{is_or_are} down. We are investigating.",
+    investigating: "We are investigating the cause and will post updates here.",
+    back_up: "%{services} %{is_or_are} back up and operating normally."
+  }
+
+  def self.active_statuses
+    reactive.active.map { |incident| IMPACT_STATUS.fetch(incident.impact) }
+  end
+
+  has_many :updates, -> { order(created_at: :desc) }, class_name: "Upright::IncidentUpdate", inverse_of: :incident, dependent: :destroy
+  has_many :affected_services, class_name: "Upright::IncidentAffectedService", inverse_of: :incident, dependent: :destroy
+  has_one :automatic_report, class_name: "Upright::Incident::AutomaticReport", inverse_of: :incident, dependent: :destroy
+
+  # An incident is public-facing when it affects at least one public service.
+  # Everything else — internal-only or untagged — stays off the public status
+  # page entirely (fail closed).
+  scope :public_facing, -> { for_service(Upright::Service.public_facing.map(&:code)).distinct }
+
+  validates :title, :starts_at, presence: true
+  validates :status, inclusion: { in: ->(incident) { incident.class::STATUSES } }
+  validates :impact, inclusion: { in: ->(incident) { incident.class::IMPACTS } }
+
+  before_validation :set_default_status, on: :create
+  before_create { self.created_by ||= Upright::Current.user&.name }
+  before_update { self.updated_by = Upright::Current.user.name if Upright::Current.user }
+  after_create :record_initial_update
+
+  def maintenance? = false
+
+  def service_codes = affected_services.map(&:service_code)
+
+  def service_codes=(codes)
+    self.affected_services = Array(codes).reject(&:blank?).uniq.map do |code|
+      affected_services.find { |s| s.service_code == code } ||
+        affected_services.build(service_code: code)
+    end
+  end
+
+  def services
+    service_codes.filter_map { |code| Upright::Service.find_by(code: code) }
+  end
+
+  def public_services
+    services.select { |service| service[:public] }
+  end
+
+  def update_body_for(key)
+    subjects = services
+    template = subjects.filter_map { |service| service.incident_update_template(key) }.uniq
+    template = template.one? ? template.first : TEMPLATES.fetch(key)
+
+    template % {
+      services: subjects.any? ? subjects.map(&:name).to_sentence : "This service",
+      is_or_are: subjects.many? ? "are" : "is"
+    }
+  end
+
+  private
+    def set_default_status
+      self.status ||= self.class::STATUSES.first
+    end
+
+    def record_initial_update
+      updates.create!(status: status, body: body.presence || default_initial_body)
+    end
+
+    def default_initial_body
+      maintenance? ? "Maintenance scheduled." : "We are investigating."
+    end
+end

data/app/models/upright/incident_affected_service.rb

@@ -0,0 +1,9 @@
+class Upright::IncidentAffectedService < Upright::PersistentRecord
+  belongs_to :incident, class_name: "Upright::Incident", inverse_of: :affected_services
+
+  validates :service_code, presence: true,
+    uniqueness: { scope: :incident_id },
+    inclusion: { in: ->(_) { Upright::Service.all.map(&:code) } }
+
+  def service = Upright::Service.find_by(code: service_code)
+end

data/app/models/upright/incident_update.rb

@@ -0,0 +1,7 @@
+class Upright::IncidentUpdate < Upright::PersistentRecord
+  belongs_to :incident, class_name: "Upright::Incident", inverse_of: :updates
+
+  validates :status, presence: true
+
+  before_create { self.created_by ||= Upright::Current.user&.name || "System" }
+end

data/app/models/upright/maintenance.rb

@@ -0,0 +1,36 @@
+class Upright::Maintenance < Upright::Incident
+  STATUSES          = %w[ scheduled in_progress completed ]
+  TERMINAL_STATUSES = %w[ completed ]
+  IMPACTS           = %w[ maintenance ]
+
+  SUPPRESSION_LEAD = 1.minute
+
+  scope :suppressing, -> { unresolved.where(starts_at: ..SUPPRESSION_LEAD.from_now) }
+
+  validates :ends_at, presence: true
+  validate :ends_after_start
+
+  before_validation :set_maintenance_impact
+
+  def maintenance? = true
+
+  def self.export_service_metrics
+    Upright::Service.all.each do |service|
+      Yabeda.upright_service_under_maintenance.set({ probe_service: service.code }, suppressing.for_service(service.code).exists? ? 1 : 0)
+    end
+  end
+
+  def auto_advance_status(now: Time.current)
+    record_update(status: "in_progress", body: "Maintenance is underway.") if scheduled? && now >= starts_at
+    record_update(status: "completed",  body: "Maintenance is complete.")  if in_progress? && now >= ends_at
+  end
+
+  private
+    def set_maintenance_impact
+      self.impact = "maintenance"
+    end
+
+    def ends_after_start
+      errors.add(:ends_at, "must be after the start") if ends_at.present? && starts_at.present? && ends_at <= starts_at
+    end
+end

data/app/models/upright/persistent_record.rb

@@ -0,0 +1,12 @@
+# Durable, replicated source-of-truth data (rollups, later incidents/maintenance) on the persistent DB.
+class Upright::PersistentRecord < ActiveRecord::Base
+  self.abstract_class = true
+
+  connects_to database: { writing: :persistent, reading: :persistent }
+
+  def self.up?
+    with_connection { |connection| connection.select_value("SELECT 1") }.present?
+  rescue StandardError
+    false
+  end
+end

data/app/models/upright/playwright/authenticator/base.rb

@@ -7,14 +7,24 @@ class Upright::Playwright::Authenticator::Base
     new.authenticate_on(page)
   end
 
-  def initialize(browser = nil, context_options = {})
-    @browser = browser
-    @context_options = context_options
+  def initialize
     @storage_state = Upright::Playwright::StorageState.new(service_name)
   end
 
+  def ensure_authenticated(context, page)
+    @page = page
+    load_cached_storage_state(context)
+    page.goto(auth_check_url, timeout: 10.seconds.in_ms)
+
+    unless session_valid_on?(page)
+      authenticate_on(page)
+      @storage_state.save(context.storage_state)
+    end
+  end
+
   def authenticate_on(page)
     @page = page
+    setup_page_logging(page)
     authenticate
     self
   end
@@ -30,14 +40,9 @@ class Upright::Playwright::Authenticator::Base
     end
   end
 
-  def authenticated_context
-    if (cached_state = @storage_state.load)
-      context = create_context(cached_state)
-      return context if context_has_valid_session?(context)
-      context.close
-    end
-
-    perform_authentication
+  def session_valid_on?(page)
+    wait_for_network_idle(page)
+    !page.url.include?(signin_path)
   end
 
   protected
@@ -50,79 +55,54 @@ class Upright::Playwright::Authenticator::Base
     raise NotImplementedError
   end
 
-  private
-
   def service_name
     raise NotImplementedError
   end
 
-  def authenticate
-    raise NotImplementedError
-  end
-
-  def context_has_valid_session?(context)
-    page = context.new_page
-    page.goto(signin_redirect_url, timeout: 10.seconds.in_ms)
-    !page.url.include?(signin_path)
-  rescue ::Playwright::TimeoutError
-    false
-  ensure
-    page&.close
-  end
-
-  def perform_authentication
-    context = create_context
-    @page = context.new_page
-    setup_page_logging(page)
-
-    authenticate
-
-    state = context.storage_state
-    @storage_state.save(state)
-    context.close
-
-    create_context(state)
-  end
+  private
+    def auth_check_url
+      signin_redirect_url
+    end
 
-  def user_agent
-    Upright.configuration.user_agent.presence ||
-      Upright::Playwright::Lifecycle::DEFAULT_USER_AGENT
-  end
+    def authenticate
+      raise NotImplementedError
+    end
 
-  def create_context(state = nil)
-    options = { userAgent: user_agent, serviceWorkers: "block" }
-    options[:storageState] = state if state
-    options.merge!(@context_options)
-    @browser.new_context(**options)
-  end
+    def load_cached_storage_state(context)
+      if (cached_state = @storage_state.load)
+        cached_state.fetch("cookies", []).each do |cookie|
+          context.add_cookies([ cookie ])
+        end
+      end
+    end
 
-  def setup_page_logging(page)
-    if defined?(RailsStructuredLogging::Recorder)
-      RailsStructuredLogging::Recorder.instance.messages.tap do |messages|
+    def setup_page_logging(page)
+      if defined?(RailsStructuredLogging::Recorder)
+        RailsStructuredLogging::Recorder.instance.messages.tap do |messages|
+          page.on("response", ->(response) {
+            next if skip_logging?(response)
+            RailsStructuredLogging::Recorder.instance.sharing(messages)
+            log_response(response)
+          })
+        end
+      else
         page.on("response", ->(response) {
           next if skip_logging?(response)
-          RailsStructuredLogging::Recorder.instance.sharing(messages)
           log_response(response)
         })
       end
-    else
-      page.on("response", ->(response) {
-        next if skip_logging?(response)
-        log_response(response)
-      })
     end
-  end
 
-  def log_response(response)
-    headers = response.headers.slice("x-request-id", "x-runtime").compact
-    Rails.logger.info "#{response.status} #{response.request.resource_type.upcase} #{response.url} #{headers.to_query}"
-  end
+    def log_response(response)
+      headers = response.headers.slice("x-request-id", "x-runtime").compact
+      Rails.logger.info "#{response.status} #{response.request.resource_type.upcase} #{response.url} #{headers.to_query}"
+    end
 
-  def skip_logging?(response)
-    %w[image asset avatar].any? { |pattern| response.url.include?(pattern) }
-  end
+    def skip_logging?(response)
+      %w[image asset avatar].any? { |pattern| response.url.include?(pattern) }
+    end
 
-  def credentials
-    Rails.application.credentials
-  end
+    def credentials
+      Rails.application.credentials
+    end
 end

data/app/models/upright/probe_result.rb

@@ -1,11 +1,16 @@
 class Upright::ProbeResult < Upright::ApplicationRecord
   include Upright::ExceptionRecording
+  include Upright::ProbeResult::StaleCleanup
+
+  attr_accessor :probe_alert_severity
 
   has_many_attached :artifacts
 
   scope :by_type,   ->(type) { where(probe_type: type) if type.present? }
   scope :by_status, ->(status) { where(status: status) if status.present? }
   scope :by_name,   ->(name) { where(probe_name: name) if name.present? }
+  scope :by_date,   ->(date) { where(created_at: Date.parse(date).all_day) if date.present? }
+
   scope :stale,     -> { where(created_at: ...24.hours.ago) }
 
   enum :status, [ :ok, :fail ]
@@ -23,7 +28,7 @@ class Upright::ProbeResult < Upright::ApplicationRecord
 
   private
     def increment_metrics
-      labels = { type: probe_type, name: probe_name, probe_target: probe_target, probe_service: probe_service }
+      labels = { type: probe_type, name: probe_name, probe_target: probe_target, probe_service: probe_service, alert_severity: probe_alert_severity || :high }
 
       Yabeda.upright_probe_duration_seconds.set(labels.merge(status: status), duration.to_f)
       Yabeda.upright_probe_up.set(labels, ok? ? 1 : 0)

data/app/models/upright/probes/http_probe.rb

@@ -6,6 +6,11 @@ class Upright::Probes::HTTPProbe < FrozenRecord::Base
 
   DEFAULT_EXPECTED_STATUS = 200..399
 
+  # Response bodies are stored as viewable/downloadable artifacts: cap their
+  # size and never store them as text/html, which would be same-origin XSS.
+  MAX_STORED_BODY_BYTES = 1.megabyte
+  TRUNCATION_NOTICE = "\n\n[TRUNCATED: response body exceeded #{MAX_STORED_BODY_BYTES} bytes]".freeze
+
   attr_accessor :last_response
 
   def check
@@ -100,7 +105,31 @@ class Upright::Probes::HTTPProbe < FrozenRecord::Base
 
     def attach_response_body(probe_result)
       if last_response && last_response.body.present?
-        Upright::Artifact.new(name: "response.#{last_response.file_extension}", content: last_response.body).attach_to(probe_result)
+        # Attach directly with identify: false — the body is untrusted remote
+        # content, and ActiveStorage's content sniffing would otherwise restore
+        # text/html and make the stored artifact same-origin XSS when previewed.
+        probe_result.artifacts.attach(
+          io: StringIO.new(stored_body),
+          filename: "response.#{stored_body_extension}",
+          content_type: stored_body_content_type,
+          identify: false
+        )
+      end
+    end
+
+    def stored_body_extension
+      last_response.file_extension == "html" ? "txt" : last_response.file_extension
+    end
+
+    def stored_body_content_type
+      Marcel::MimeType.for(extension: stored_body_extension)
+    end
+
+    def stored_body
+      if last_response.body.bytesize > MAX_STORED_BODY_BYTES
+        last_response.body.byteslice(0, MAX_STORED_BODY_BYTES) + TRUNCATION_NOTICE
+      else
+        last_response.body
       end
     end
 end

data/app/models/upright/probes/playwright/base.rb

@@ -11,6 +11,7 @@ class Upright::Probes::Playwright::Base
   include Upright::Playwright::Logging
   include Upright::Playwright::OtelTracing
   include Upright::Playwright::VideoRecording
+  include Upright::Playwright::TraceRecording
   include Upright::Playwright::Helpers
 
   set_callback :perform_check, :after, :wait_for_network_idle
@@ -39,6 +40,7 @@ class Upright::Probes::Playwright::Base
 
   def on_check_recorded(probe_result)
     attach_video(probe_result)
+    attach_trace(probe_result)
     attach_log(probe_result)
   end
 

data/app/models/upright/probes/status/probe.rb

@@ -10,6 +10,14 @@ class Upright::Probes::Status::Probe
     @site_statuses = site_statuses
   end
 
+  def self.key_for(name:, type:, probe_target:, **)
+    [ name, type, probe_target ]
+  end
+
+  def key
+    self.class.key_for(name:, type:, probe_target:)
+  end
+
   def status_for_site(code)
     site_statuses.find { |s| s.site_code == code.to_s }
   end

data/app/models/upright/probes/status.rb

@@ -1,7 +1,7 @@
 class Upright::Probes::Status
   class << self
     def for_type(probe_type)
-      results = prometheus_client.query_range(
+      results = Upright.prometheus_client.query_range(
         query: query(probe_type),
         start: 30.minutes.ago.iso8601,
         end:   Time.current.iso8601,
@@ -17,21 +17,13 @@ class Upright::Probes::Status
       end
 
       def label_selector(probe_type)
-        if probe_type.present?
-          "{type=\"#{probe_type}\"}"
-        end
-      end
-
-      def prometheus_client
-        Prometheus::ApiClient.client(
-          url: ENV.fetch("PROMETHEUS_URL", "http://localhost:9090"),
-          options: { timeout: 30.seconds }
-        )
+        matchers = [ "alert_severity!=\"\"", "environment=\"#{Rails.env}\"" ]
+        matchers << "type=\"#{probe_type}\"" if probe_type.present?
+        "{#{matchers.join(",")}}"
       end
 
       def build_probes(results)
-        # Group results by probe identity (name + type + probe_target)
-        grouped = results.group_by { |r| [ r[:metric][:name], r[:metric][:type], r[:metric][:probe_target] ] }
+        grouped = results.group_by { |r| Probe.key_for(**r[:metric]) }
 
         grouped.map do |(_name, _type, _target), series|
           site_statuses = series.map do |s|

data/app/models/upright/probes/uptime.rb

@@ -5,7 +5,7 @@ class Upright::Probes::Uptime
     end
 
     def for_type(probe_type)
-      results = prometheus_client.query_range(
+      results = Upright.prometheus_client.query_range(
         query: query(probe_type),
         start: 30.days.ago.iso8601,
         end:   Time.current.iso8601,
@@ -17,20 +17,13 @@ class Upright::Probes::Uptime
 
     private
       def query(probe_type)
-        "upright:probe_uptime_daily#{label_selector(probe_type)}"
+        "min by (name, type, probe_target) (upright:probe_uptime_daily#{label_selector(probe_type)})"
       end
 
       def label_selector(probe_type)
-        if probe_type.present?
-          "{type=\"#{probe_type}\"}"
-        end
-      end
-
-      def prometheus_client
-        Prometheus::ApiClient.client(
-          url: ENV.fetch("PROMETHEUS_URL", "http://localhost:9090"),
-          options: { timeout: 30.seconds }
-        )
+        matchers = [ "alert_severity!=\"\"", "environment=\"#{Rails.env}\"" ]
+        matchers << "type=\"#{probe_type}\"" if probe_type.present?
+        "{#{matchers.join(",")}}"
       end
   end
 end

data/app/models/upright/rollups/daily_uptime.rb

@@ -0,0 +1,28 @@
+class Upright::Rollups::DailyUptime
+  attr_reader :day
+
+  def initialize(day, sites: Upright.sites.select(&:stores_metrics?))
+    @day, @sites = day, sites
+  end
+
+  def covered
+    probe_uptimes.select(&:covered?)
+  end
+
+  def gappy
+    probe_uptimes.reject(&:covered?)
+  end
+
+  private
+    def probe_uptimes
+      @probe_uptimes ||= best_covered_per_probe(reported_uptimes)
+    end
+
+    def best_covered_per_probe(probe_uptimes)
+      probe_uptimes.group_by(&:probe_key).values.map { |reported| reported.max_by(&:coverage_fraction) }
+    end
+
+    def reported_uptimes
+      @sites.flat_map { |site| Upright::Rollups::SiteUptime.new(site, on: day).probe_uptimes }
+    end
+end

data/app/models/upright/rollups/probe_rollup.rb

@@ -0,0 +1,64 @@
+class Upright::Rollups::ProbeRollup < Upright::PersistentRecord
+  self.table_name = "upright_rollups_probe_rollups"
+
+  enum :status, Upright::Status::VALUES, default: :operational
+
+  before_save :derive_status_from_uptime
+
+  scope :for_period, ->(range) { where(period_start: range) }
+  scope :for_service, ->(code) { where(probe_service: code) if code.present? }
+  scope :for_probe, ->(name) { where(probe_name: name) if name.present? }
+
+  def self.rollup_day(day)
+    uptime = Upright::Rollups::DailyUptime.new(day)
+
+    uptime.covered.each { |probe_uptime| record probe_uptime, on: day }
+
+    uptime.gappy.tap do |left_unwritten|
+      Rails.logger.warn thin_coverage_warning(left_unwritten, on: day) if left_unwritten.any?
+    end
+  end
+
+  def self.thin_coverage_warning(probe_uptimes, on:)
+    "[upright] #{on}: left #{probe_uptimes.size} rollups unwritten below #{Upright.configuration.rollup_minimum_coverage} coverage (#{probe_uptimes.map(&:probe_name).uniq.join(', ')})"
+  end
+  private_class_method :thin_coverage_warning
+
+  def self.record(probe_uptime, on:)
+    rollup = find_or_initialize_by(
+      probe_name:   probe_uptime.probe_name,
+      probe_type:   probe_uptime.probe_type,
+      probe_target: probe_uptime.probe_target,
+      period_start: on.beginning_of_day
+    )
+
+    rollup.probe_service   = probe_uptime.probe_service
+    rollup.uptime_fraction = probe_uptime.uptime_fraction
+    rollup.save!
+  end
+
+  def self.export_metrics
+    if (last_run = maximum(:created_at))
+      Yabeda.upright_rollup_last_run_timestamp_seconds.set({}, last_run.to_i)
+    end
+  end
+
+  def probe_key
+    [ probe_name, probe_type, probe_target ]
+  end
+
+  def service
+    Upright::Service.find_by(code: probe_service) if probe_service.present?
+  end
+
+  def uptime_percentage
+    if uptime_fraction.present?
+      (uptime_fraction * 100).round(4)
+    end
+  end
+
+  private
+    def derive_status_from_uptime
+      self.status = Upright::Status.for(uptime_fraction)
+    end
+end

data/app/models/upright/rollups/probe_uptime.rb

@@ -0,0 +1,9 @@
+class Upright::Rollups::ProbeUptime < Data.define(:probe_name, :probe_type, :probe_target, :probe_service, :uptime_fraction, :coverage_fraction)
+  def probe_key
+    [ probe_name, probe_type, probe_target ]
+  end
+
+  def covered?
+    coverage_fraction >= Upright.configuration.rollup_minimum_coverage
+  end
+end

data/app/models/upright/rollups/site_uptime.rb

@@ -0,0 +1,75 @@
+class Upright::Rollups::SiteUptime
+  UPTIME_METRIC   = "upright:probe_uptime_daily"
+  COVERAGE_METRIC = "upright:probe_down_fraction"
+
+  attr_reader :site, :day
+
+  def initialize(site, on:)
+    @site, @day = site, on
+  end
+
+  def probe_uptimes
+    uptime_series.map do |metric, uptime|
+      Upright::Rollups::ProbeUptime.new(
+        probe_name:        metric[:name],
+        probe_type:        metric[:type],
+        probe_target:      metric[:probe_target],
+        probe_service:     metric[:probe_service].presence,
+        uptime_fraction:   uptime,
+        coverage_fraction: coverage_for(metric)
+      )
+    end
+  rescue StandardError => error
+    nothing_reported because: error
+  end
+
+  private
+    def nothing_reported(because:)
+      Rails.logger.warn "[upright] #{site.code} Prometheus unavailable for the #{day} rollup: #{because.class}: #{because.message}"
+      []
+    end
+
+    def uptime_series
+      series_for uptime_query
+    end
+
+    def coverage_for(metric)
+      [ coverage.fetch(probe_key_for(metric), 0.0), 1.0 ].min
+    end
+
+    def coverage
+      @coverage ||= series_for(coverage_query).to_h do |metric, samples|
+        [ probe_key_for(metric), samples / expected_samples ]
+      end
+    end
+
+    def series_for(query)
+      values_by_metric site.prometheus_client.query(query: query, time: queried_at.iso8601)
+    end
+
+    def values_by_metric(response)
+      Array(response.deep_symbolize_keys[:result]).map do |series|
+        [ series[:metric], series.dig(:value, 1).to_f ]
+      end
+    end
+
+    def queried_at
+      [ day.end_of_day, Time.current ].min
+    end
+
+    def expected_samples
+      1.day / Upright.configuration.rollup_evaluation_interval
+    end
+
+    def probe_key_for(metric)
+      metric.values_at(:name, :type, :probe_target)
+    end
+
+    def uptime_query
+      %(#{UPTIME_METRIC}{environment="#{Rails.env}"})
+    end
+
+    def coverage_query
+      %(count_over_time(#{COVERAGE_METRIC}{environment="#{Rails.env}"}[1d]))
+    end
+end