upright 0.3.0 → 0.4.0

data/app/models/upright/maintenance.rb

@@ -0,0 +1,36 @@
+class Upright::Maintenance < Upright::Incident
+  STATUSES          = %w[ scheduled in_progress completed ]
+  TERMINAL_STATUSES = %w[ completed ]
+  IMPACTS           = %w[ maintenance ]
+
+  SUPPRESSION_LEAD = 1.minute
+
+  scope :suppressing, -> { unresolved.where(starts_at: ..SUPPRESSION_LEAD.from_now) }
+
+  validates :ends_at, presence: true
+  validate :ends_after_start
+
+  before_validation :set_maintenance_impact
+
+  def maintenance? = true
+
+  def self.export_service_metrics
+    Upright::Service.all.each do |service|
+      Yabeda.upright_service_under_maintenance.set({ probe_service: service.code }, suppressing.for_service(service.code).exists? ? 1 : 0)
+    end
+  end
+
+  def auto_advance_status(now: Time.current)
+    record_update(status: "in_progress", body: "Maintenance is underway.") if scheduled? && now >= starts_at
+    record_update(status: "completed",  body: "Maintenance is complete.")  if in_progress? && now >= ends_at
+  end
+
+  private
+    def set_maintenance_impact
+      self.impact = "maintenance"
+    end
+
+    def ends_after_start
+      errors.add(:ends_at, "must be after the start") if ends_at.present? && starts_at.present? && ends_at <= starts_at
+    end
+end

data/app/models/upright/persistent_record.rb

@@ -0,0 +1,12 @@
+# Durable, replicated source-of-truth data (rollups, later incidents/maintenance) on the persistent DB.
+class Upright::PersistentRecord < ActiveRecord::Base
+  self.abstract_class = true
+
+  connects_to database: { writing: :persistent, reading: :persistent }
+
+  def self.up?
+    with_connection { |connection| connection.select_value("SELECT 1") }.present?
+  rescue StandardError
+    false
+  end
+end

data/app/models/upright/playwright/authenticator/base.rb

@@ -7,14 +7,24 @@ class Upright::Playwright::Authenticator::Base
     new.authenticate_on(page)
   end
 
-  def initialize(browser = nil, context_options = {})
-    @browser = browser
-    @context_options = context_options
+  def initialize
     @storage_state = Upright::Playwright::StorageState.new(service_name)
   end
 
+  def ensure_authenticated(context, page)
+    @page = page
+    load_cached_storage_state(context)
+    page.goto(auth_check_url, timeout: 10.seconds.in_ms)
+
+    unless session_valid_on?(page)
+      authenticate_on(page)
+      @storage_state.save(context.storage_state)
+    end
+  end
+
   def authenticate_on(page)
     @page = page
+    setup_page_logging(page)
     authenticate
     self
   end
@@ -30,14 +40,9 @@ class Upright::Playwright::Authenticator::Base
     end
   end
 
-  def authenticated_context
-    if (cached_state = @storage_state.load)
-      context = create_context(cached_state)
-      return context if context_has_valid_session?(context)
-      context.close
-    end
-
-    perform_authentication
+  def session_valid_on?(page)
+    wait_for_network_idle(page)
+    !page.url.include?(signin_path)
   end
 
   protected
@@ -50,79 +55,54 @@ class Upright::Playwright::Authenticator::Base
     raise NotImplementedError
   end
 
-  private
-
   def service_name
     raise NotImplementedError
   end
 
-  def authenticate
-    raise NotImplementedError
-  end
-
-  def context_has_valid_session?(context)
-    page = context.new_page
-    page.goto(signin_redirect_url, timeout: 10.seconds.in_ms)
-    !page.url.include?(signin_path)
-  rescue ::Playwright::TimeoutError
-    false
-  ensure
-    page&.close
-  end
-
-  def perform_authentication
-    context = create_context
-    @page = context.new_page
-    setup_page_logging(page)
-
-    authenticate
-
-    state = context.storage_state
-    @storage_state.save(state)
-    context.close
-
-    create_context(state)
-  end
+  private
+    def auth_check_url
+      signin_redirect_url
+    end
 
-  def user_agent
-    Upright.configuration.user_agent.presence ||
-      Upright::Playwright::Lifecycle::DEFAULT_USER_AGENT
-  end
+    def authenticate
+      raise NotImplementedError
+    end
 
-  def create_context(state = nil)
-    options = { userAgent: user_agent, serviceWorkers: "block" }
-    options[:storageState] = state if state
-    options.merge!(@context_options)
-    @browser.new_context(**options)
-  end
+    def load_cached_storage_state(context)
+      if (cached_state = @storage_state.load)
+        cached_state.fetch("cookies", []).each do |cookie|
+          context.add_cookies([ cookie ])
+        end
+      end
+    end
 
-  def setup_page_logging(page)
-    if defined?(RailsStructuredLogging::Recorder)
-      RailsStructuredLogging::Recorder.instance.messages.tap do |messages|
+    def setup_page_logging(page)
+      if defined?(RailsStructuredLogging::Recorder)
+        RailsStructuredLogging::Recorder.instance.messages.tap do |messages|
+          page.on("response", ->(response) {
+            next if skip_logging?(response)
+            RailsStructuredLogging::Recorder.instance.sharing(messages)
+            log_response(response)
+          })
+        end
+      else
         page.on("response", ->(response) {
           next if skip_logging?(response)
-          RailsStructuredLogging::Recorder.instance.sharing(messages)
           log_response(response)
         })
       end
-    else
-      page.on("response", ->(response) {
-        next if skip_logging?(response)
-        log_response(response)
-      })
     end
-  end
 
-  def log_response(response)
-    headers = response.headers.slice("x-request-id", "x-runtime").compact
-    Rails.logger.info "#{response.status} #{response.request.resource_type.upcase} #{response.url} #{headers.to_query}"
-  end
+    def log_response(response)
+      headers = response.headers.slice("x-request-id", "x-runtime").compact
+      Rails.logger.info "#{response.status} #{response.request.resource_type.upcase} #{response.url} #{headers.to_query}"
+    end
 
-  def skip_logging?(response)
-    %w[image asset avatar].any? { |pattern| response.url.include?(pattern) }
-  end
+    def skip_logging?(response)
+      %w[image asset avatar].any? { |pattern| response.url.include?(pattern) }
+    end
 
-  def credentials
-    Rails.application.credentials
-  end
+    def credentials
+      Rails.application.credentials
+    end
 end

data/app/models/upright/probes/http_probe.rb

@@ -6,6 +6,11 @@ class Upright::Probes::HTTPProbe < FrozenRecord::Base
 
   DEFAULT_EXPECTED_STATUS = 200..399
 
+  # Response bodies are stored as viewable/downloadable artifacts: cap their
+  # size and never store them as text/html, which would be same-origin XSS.
+  MAX_STORED_BODY_BYTES = 1.megabyte
+  TRUNCATION_NOTICE = "\n\n[TRUNCATED: response body exceeded #{MAX_STORED_BODY_BYTES} bytes]".freeze
+
   attr_accessor :last_response
 
   def check
@@ -100,7 +105,31 @@ class Upright::Probes::HTTPProbe < FrozenRecord::Base
 
     def attach_response_body(probe_result)
       if last_response && last_response.body.present?
-        Upright::Artifact.new(name: "response.#{last_response.file_extension}", content: last_response.body).attach_to(probe_result)
+        # Attach directly with identify: false — the body is untrusted remote
+        # content, and ActiveStorage's content sniffing would otherwise restore
+        # text/html and make the stored artifact same-origin XSS when previewed.
+        probe_result.artifacts.attach(
+          io: StringIO.new(stored_body),
+          filename: "response.#{stored_body_extension}",
+          content_type: stored_body_content_type,
+          identify: false
+        )
+      end
+    end
+
+    def stored_body_extension
+      last_response.file_extension == "html" ? "txt" : last_response.file_extension
+    end
+
+    def stored_body_content_type
+      Marcel::MimeType.for(extension: stored_body_extension)
+    end
+
+    def stored_body
+      if last_response.body.bytesize > MAX_STORED_BODY_BYTES
+        last_response.body.byteslice(0, MAX_STORED_BODY_BYTES) + TRUNCATION_NOTICE
+      else
+        last_response.body
       end
     end
 end

data/app/models/upright/probes/playwright/base.rb

@@ -11,6 +11,7 @@ class Upright::Probes::Playwright::Base
   include Upright::Playwright::Logging
   include Upright::Playwright::OtelTracing
   include Upright::Playwright::VideoRecording
+  include Upright::Playwright::TraceRecording
   include Upright::Playwright::Helpers
 
   set_callback :perform_check, :after, :wait_for_network_idle
@@ -39,6 +40,7 @@ class Upright::Probes::Playwright::Base
 
   def on_check_recorded(probe_result)
     attach_video(probe_result)
+    attach_trace(probe_result)
     attach_log(probe_result)
   end
 

data/app/models/upright/probes/status/probe.rb

@@ -10,6 +10,14 @@ class Upright::Probes::Status::Probe
     @site_statuses = site_statuses
   end
 
+  def self.key_for(name:, type:, probe_target:, **)
+    [ name, type, probe_target ]
+  end
+
+  def key
+    self.class.key_for(name:, type:, probe_target:)
+  end
+
   def status_for_site(code)
     site_statuses.find { |s| s.site_code == code.to_s }
   end

data/app/models/upright/probes/status.rb

@@ -1,7 +1,7 @@
 class Upright::Probes::Status
   class << self
     def for_type(probe_type)
-      results = prometheus_client.query_range(
+      results = Upright.prometheus_client.query_range(
         query: query(probe_type),
         start: 30.minutes.ago.iso8601,
         end:   Time.current.iso8601,
@@ -17,21 +17,13 @@ class Upright::Probes::Status
       end
 
       def label_selector(probe_type)
-        matchers = [ "alert_severity!=\"\"" ]
+        matchers = [ "alert_severity!=\"\"", "environment=\"#{Rails.env}\"" ]
         matchers << "type=\"#{probe_type}\"" if probe_type.present?
         "{#{matchers.join(",")}}"
       end
 
-      def prometheus_client
-        Prometheus::ApiClient.client(
-          url: ENV.fetch("PROMETHEUS_URL", "http://localhost:9090"),
-          options: { timeout: 30.seconds }
-        )
-      end
-
       def build_probes(results)
-        # Group results by probe identity (name + type + probe_target)
-        grouped = results.group_by { |r| [ r[:metric][:name], r[:metric][:type], r[:metric][:probe_target] ] }
+        grouped = results.group_by { |r| Probe.key_for(**r[:metric]) }
 
         grouped.map do |(_name, _type, _target), series|
           site_statuses = series.map do |s|

data/app/models/upright/probes/uptime.rb

@@ -5,7 +5,7 @@ class Upright::Probes::Uptime
     end
 
     def for_type(probe_type)
-      results = prometheus_client.query_range(
+      results = Upright.prometheus_client.query_range(
         query: query(probe_type),
         start: 30.days.ago.iso8601,
         end:   Time.current.iso8601,
@@ -21,16 +21,9 @@ class Upright::Probes::Uptime
       end
 
       def label_selector(probe_type)
-        matchers = [ "alert_severity!=\"\"" ]
+        matchers = [ "alert_severity!=\"\"", "environment=\"#{Rails.env}\"" ]
         matchers << "type=\"#{probe_type}\"" if probe_type.present?
         "{#{matchers.join(",")}}"
       end
-
-      def prometheus_client
-        Prometheus::ApiClient.client(
-          url: ENV.fetch("PROMETHEUS_URL", "http://localhost:9090"),
-          options: { timeout: 30.seconds }
-        )
-      end
   end
 end

data/app/models/upright/rollups/daily_uptime.rb

@@ -0,0 +1,28 @@
+class Upright::Rollups::DailyUptime
+  attr_reader :day
+
+  def initialize(day, sites: Upright.sites.select(&:stores_metrics?))
+    @day, @sites = day, sites
+  end
+
+  def covered
+    probe_uptimes.select(&:covered?)
+  end
+
+  def gappy
+    probe_uptimes.reject(&:covered?)
+  end
+
+  private
+    def probe_uptimes
+      @probe_uptimes ||= best_covered_per_probe(reported_uptimes)
+    end
+
+    def best_covered_per_probe(probe_uptimes)
+      probe_uptimes.group_by(&:probe_key).values.map { |reported| reported.max_by(&:coverage_fraction) }
+    end
+
+    def reported_uptimes
+      @sites.flat_map { |site| Upright::Rollups::SiteUptime.new(site, on: day).probe_uptimes }
+    end
+end

data/app/models/upright/rollups/probe_rollup.rb

@@ -0,0 +1,64 @@
+class Upright::Rollups::ProbeRollup < Upright::PersistentRecord
+  self.table_name = "upright_rollups_probe_rollups"
+
+  enum :status, Upright::Status::VALUES, default: :operational
+
+  before_save :derive_status_from_uptime
+
+  scope :for_period, ->(range) { where(period_start: range) }
+  scope :for_service, ->(code) { where(probe_service: code) if code.present? }
+  scope :for_probe, ->(name) { where(probe_name: name) if name.present? }
+
+  def self.rollup_day(day)
+    uptime = Upright::Rollups::DailyUptime.new(day)
+
+    uptime.covered.each { |probe_uptime| record probe_uptime, on: day }
+
+    uptime.gappy.tap do |left_unwritten|
+      Rails.logger.warn thin_coverage_warning(left_unwritten, on: day) if left_unwritten.any?
+    end
+  end
+
+  def self.thin_coverage_warning(probe_uptimes, on:)
+    "[upright] #{on}: left #{probe_uptimes.size} rollups unwritten below #{Upright.configuration.rollup_minimum_coverage} coverage (#{probe_uptimes.map(&:probe_name).uniq.join(', ')})"
+  end
+  private_class_method :thin_coverage_warning
+
+  def self.record(probe_uptime, on:)
+    rollup = find_or_initialize_by(
+      probe_name:   probe_uptime.probe_name,
+      probe_type:   probe_uptime.probe_type,
+      probe_target: probe_uptime.probe_target,
+      period_start: on.beginning_of_day
+    )
+
+    rollup.probe_service   = probe_uptime.probe_service
+    rollup.uptime_fraction = probe_uptime.uptime_fraction
+    rollup.save!
+  end
+
+  def self.export_metrics
+    if (last_run = maximum(:created_at))
+      Yabeda.upright_rollup_last_run_timestamp_seconds.set({}, last_run.to_i)
+    end
+  end
+
+  def probe_key
+    [ probe_name, probe_type, probe_target ]
+  end
+
+  def service
+    Upright::Service.find_by(code: probe_service) if probe_service.present?
+  end
+
+  def uptime_percentage
+    if uptime_fraction.present?
+      (uptime_fraction * 100).round(4)
+    end
+  end
+
+  private
+    def derive_status_from_uptime
+      self.status = Upright::Status.for(uptime_fraction)
+    end
+end

data/app/models/upright/rollups/probe_uptime.rb

@@ -0,0 +1,9 @@
+class Upright::Rollups::ProbeUptime < Data.define(:probe_name, :probe_type, :probe_target, :probe_service, :uptime_fraction, :coverage_fraction)
+  def probe_key
+    [ probe_name, probe_type, probe_target ]
+  end
+
+  def covered?
+    coverage_fraction >= Upright.configuration.rollup_minimum_coverage
+  end
+end

data/app/models/upright/rollups/site_uptime.rb

@@ -0,0 +1,75 @@
+class Upright::Rollups::SiteUptime
+  UPTIME_METRIC   = "upright:probe_uptime_daily"
+  COVERAGE_METRIC = "upright:probe_down_fraction"
+
+  attr_reader :site, :day
+
+  def initialize(site, on:)
+    @site, @day = site, on
+  end
+
+  def probe_uptimes
+    uptime_series.map do |metric, uptime|
+      Upright::Rollups::ProbeUptime.new(
+        probe_name:        metric[:name],
+        probe_type:        metric[:type],
+        probe_target:      metric[:probe_target],
+        probe_service:     metric[:probe_service].presence,
+        uptime_fraction:   uptime,
+        coverage_fraction: coverage_for(metric)
+      )
+    end
+  rescue StandardError => error
+    nothing_reported because: error
+  end
+
+  private
+    def nothing_reported(because:)
+      Rails.logger.warn "[upright] #{site.code} Prometheus unavailable for the #{day} rollup: #{because.class}: #{because.message}"
+      []
+    end
+
+    def uptime_series
+      series_for uptime_query
+    end
+
+    def coverage_for(metric)
+      [ coverage.fetch(probe_key_for(metric), 0.0), 1.0 ].min
+    end
+
+    def coverage
+      @coverage ||= series_for(coverage_query).to_h do |metric, samples|
+        [ probe_key_for(metric), samples / expected_samples ]
+      end
+    end
+
+    def series_for(query)
+      values_by_metric site.prometheus_client.query(query: query, time: queried_at.iso8601)
+    end
+
+    def values_by_metric(response)
+      Array(response.deep_symbolize_keys[:result]).map do |series|
+        [ series[:metric], series.dig(:value, 1).to_f ]
+      end
+    end
+
+    def queried_at
+      [ day.end_of_day, Time.current ].min
+    end
+
+    def expected_samples
+      1.day / Upright.configuration.rollup_evaluation_interval
+    end
+
+    def probe_key_for(metric)
+      metric.values_at(:name, :type, :probe_target)
+    end
+
+    def uptime_query
+      %(#{UPTIME_METRIC}{environment="#{Rails.env}"})
+    end
+
+    def coverage_query
+      %(count_over_time(#{COVERAGE_METRIC}{environment="#{Rails.env}"}[1d]))
+    end
+end

data/app/models/upright/service/daily_status.rb

@@ -0,0 +1,39 @@
+class Upright::Service::DailyStatus
+  attr_reader :date, :status, :uptime_fraction
+
+  def initialize(date:, status: nil, uptime_fraction: nil)
+    @date            = date
+    @status          = status
+    @uptime_fraction = uptime_fraction
+  end
+
+  def operational?
+    status == :operational
+  end
+
+  def date_label
+    date == Date.current ? "Today" : date.to_fs(:month_day)
+  end
+
+  def detail
+    if uptime_fraction
+      [ "%.2f%% uptime" % (uptime_fraction * 100), downtime ].compact.join(" · ")
+    elsif status
+      status.to_s.humanize.downcase
+    else
+      "no data"
+    end
+  end
+
+  def aria_label
+    "#{date_label}: #{detail}"
+  end
+
+  private
+    def downtime
+      if uptime_fraction && uptime_fraction < 1
+        minutes = ((1 - uptime_fraction) * 24 * 60).round
+        minutes.zero? ? "<1 min down" : "#{minutes} #{'min'.pluralize(minutes)} down"
+      end
+    end
+end

data/app/models/upright/service/incident_history.rb

@@ -0,0 +1,24 @@
+class Upright::Service::IncidentHistory
+  attr_reader :service, :page
+
+  delegate :daily_status_history, :maintenance_active?, :name, to: :service
+
+  def self.for(service, page: nil)
+    new service, page: GearedPagination::Recordset.new(
+      service.incidents.past.reorder(nil).preload(:updates), ordered_by: { starts_at: :desc }
+    ).page(page)
+  end
+
+  def initialize(service, page:)
+    @service = service
+    @page = page
+  end
+
+  def active_incidents
+    @active_incidents ||= service.incidents.reactive.active.preload(:updates)
+  end
+
+  def past_incidents
+    page.records
+  end
+end

data/app/models/upright/service/outage.rb

@@ -0,0 +1,15 @@
+class Upright::Service::Outage < Data.define(:service, :status, :started_at)
+  IMPACT_BY_STATUS = {
+    degraded: "minor",
+    partial_outage: "major",
+    major_outage: "critical"
+  }
+
+  def impact
+    IMPACT_BY_STATUS.fetch(status)
+  end
+
+  def started_at_or_lookback(observed_at:)
+    started_at || observed_at - Upright::Services::LiveStatus::OUTAGE_LOOKBACK
+  end
+end

data/app/models/upright/service/status_page.rb

@@ -0,0 +1,35 @@
+class Upright::Service::StatusPage
+  attr_reader :services, :active_incidents, :active_maintenances, :upcoming_maintenances
+
+  def self.current
+    services = Upright::Service.public_facing
+
+    new services: services,
+      active_incidents: Upright::Incident.public_facing.reactive.active.order(starts_at: :desc),
+      active_maintenances: Upright::Maintenance.public_facing.active.order(:starts_at),
+      upcoming_maintenances: Upright::Maintenance.public_facing.upcoming.order(:starts_at)
+  end
+
+  def initialize(services:, active_incidents:, active_maintenances:, upcoming_maintenances:)
+    @services = services
+    @active_incidents = active_incidents
+    @active_maintenances = active_maintenances
+    @upcoming_maintenances = upcoming_maintenances
+  end
+
+  def overall_status
+    services.overall_status(incidents: Upright::Incident.public_facing)
+  end
+
+  def outages
+    services.degraded
+  end
+
+  def active_events
+    active_incidents.to_a + active_maintenances.to_a
+  end
+
+  def service_histories
+    services.by_history
+  end
+end