upright 0.3.0 → 0.4.0

data/app/models/concerns/upright/incidents/auto_reporting.rb

@@ -0,0 +1,93 @@
+module Upright::Incidents::AutoReporting
+  extend ActiveSupport::Concern
+
+  RESOLVE_DELAY = 5.minutes
+
+  included do
+    scope :auto_created, -> { where(auto_created: true) }
+  end
+
+  class_methods do
+    def report_downtime(now: Time.current)
+      outages = Upright::Service.degraded
+
+      outages.each { |outage| report_outage(outage, observed_at: now) }
+      observe_recoveries(excluding: outages.map(&:service), observed_at: now)
+    end
+
+    private
+      def report_outage(outage, observed_at:)
+        if incident = reactive.unresolved.for_service(outage.service.code).first
+          incident.observe_downtime(outage, at: observed_at)
+        else
+          open_automatically(outage, at: observed_at)
+        end
+      rescue ActiveRecord::RecordNotUnique
+        joins(:automatic_report)
+          .find_by!(upright_incident_automatic_reports: { service_code: outage.service.code })
+          .observe_downtime(outage, at: observed_at)
+      end
+
+      def open_automatically(outage, at:)
+        new(
+          title: "#{outage.service.name} outage",
+          impact: outage.impact,
+          starts_at: outage.started_at_or_lookback(observed_at: at),
+          last_seen_down_at: at,
+          created_by: "System"
+        ).tap do |incident|
+          incident.automatically_affect(outage.service)
+          incident.body = incident.update_body_for(:down)
+          incident.save!
+        end
+      end
+
+      def observe_recoveries(excluding:, observed_at:)
+        degraded_codes = excluding.map(&:code)
+
+        auto_created.reactive.unresolved.find_each do |incident|
+          incident.observe_recovery(at: observed_at) unless (incident.service_codes & degraded_codes).any?
+        end
+      end
+  end
+
+  def automatically_affect(service)
+    self.auto_created = true
+    self.service_codes = [ service.code ]
+    build_automatic_report(service_code: service.code)
+  end
+
+  def observe_downtime(outage, at:)
+    self.last_seen_down_at = at
+    self.recovery_started_at = nil
+    escalate_to(outage.impact)
+    save!
+  end
+
+  def observe_recovery(at:)
+    if services.any?(&:maintenance_active?)
+      pause_recovery
+    elsif recovery_started_at.nil?
+      update!(recovery_started_at: at)
+    elsif recovery_started_at <= at - RESOLVE_DELAY
+      resolve_automatically(at: at)
+    end
+  end
+
+  private
+    def escalate_to(new_impact)
+      self.impact = new_impact if self.class::IMPACTS.index(new_impact) > self.class::IMPACTS.index(impact)
+    end
+
+    def pause_recovery
+      update!(recovery_started_at: nil) if recovery_started_at?
+    end
+
+    def resolve_automatically(at:)
+      record_update({ status: "resolved", body: update_body_for(:back_up) }, recorded_at: at)
+    end
+
+    def after_recording_terminal_status
+      automatic_report&.destroy!
+    end
+end

data/app/models/concerns/upright/incidents/lifecycle.rb

@@ -0,0 +1,47 @@
+module Upright::Incidents::Lifecycle
+  extend ActiveSupport::Concern
+
+  included do
+    scope :resolved,   -> { where.not(resolved_at: nil) }
+    scope :unresolved, -> { where(resolved_at: nil) }
+
+    scope :active,   -> { unresolved.where(starts_at: ..Time.current) }
+    scope :upcoming, -> { unresolved.where(starts_at: Time.current..) }
+    scope :past,     -> { resolved.order(starts_at: :desc) }
+
+    scope :reactive, -> { where.not(type: "Upright::Maintenance").or(where(type: nil)) }
+    scope :planned,  -> { where(type: "Upright::Maintenance") }
+
+    scope :for_service, ->(code) {
+      joins(:affected_services).where(upright_incident_affected_services: { service_code: code })
+    }
+  end
+
+  def active?   = resolved_at.nil? && starts_at <= Time.current
+  def upcoming? = resolved_at.nil? && starts_at > Time.current
+  def past?     = resolved_at.present?
+
+  def record_update(attributes = nil, recorded_at: Time.current, **update_attributes)
+    attributes = attributes ? attributes.merge(update_attributes) : update_attributes
+
+    updates.build(attributes.merge(created_at: recorded_at)).tap do |update|
+      next unless update.valid?
+
+      self.status = update.status
+      if self.class::TERMINAL_STATUSES.include?(update.status)
+        self.resolved_at ||= recorded_at
+      end
+      next unless valid?
+
+      transaction do
+        update.save
+        save
+        after_recording_terminal_status if self.class::TERMINAL_STATUSES.include?(update.status)
+      end
+    end
+  end
+
+  private
+    def after_recording_terminal_status
+    end
+end

data/app/models/concerns/upright/playwright/form_authentication.rb

@@ -3,6 +3,7 @@ module Upright::Playwright::FormAuthentication
 
   included do
     class_attribute :authentication_service
+    set_callback :page_ready, :after, :ensure_authenticated
   end
 
   class_methods do
@@ -12,9 +13,9 @@ module Upright::Playwright::FormAuthentication
   end
 
   private
-    def authenticated_context(browser, context_options = {})
+    def ensure_authenticated
       if authentication_service
-        authenticator_for(authentication_service).new(browser, context_options).authenticated_context
+        authenticator_for(authentication_service).new.ensure_authenticated(context, page)
       end
     end
 

data/app/models/concerns/upright/playwright/lifecycle.rb

@@ -16,14 +16,8 @@ module Upright::Playwright::Lifecycle
 
   private
     def with_browser(&block)
-      if ENV["LOCAL_PLAYWRIGHT"]
-        ::Playwright.create(playwright_cli_executable_path: "./node_modules/.bin/playwright") do |playwright|
-          playwright.chromium.launch(headless: false, &block)
-        end
-      else
-        server_url = Upright.configuration.playwright_server_url ||
-                     Rails.application.config_for(:playwright).fetch(:server_url)
-        ::Playwright.connect_to_browser_server(server_url, &block)
+      ::Playwright.create(playwright_cli_executable_path: Upright.configuration.playwright_cli_path) do |playwright|
+        playwright.chromium.launch(headless: ENV.fetch("HEADLESS", "true") != "false", &block)
       end
     end
 
@@ -33,13 +27,14 @@ module Upright::Playwright::Lifecycle
       run_callbacks :page_ready
       yield
     ensure
-      # Rescue each step independently so a failed close doesn't prevent video capture
-      page&.close rescue Rails.error.report($!)
-      context&.close rescue Rails.error.report($!)
-      run_callbacks :page_close
+      run_callbacks :page_close do
+        # Rescue each step independently so one failed close does not block artifact capture
+        page&.close rescue Rails.error.report($!)
+        context&.close rescue Rails.error.report($!)
+      end
     end
 
     def create_context(browser, **options)
-      authenticated_context(browser, options) || browser.new_context(userAgent: user_agent, **options)
+      browser.new_context(userAgent: user_agent, **options)
     end
 end

data/app/models/concerns/upright/playwright/trace_recording.rb

@@ -0,0 +1,53 @@
+module Upright::Playwright::TraceRecording
+  extend ActiveSupport::Concern
+
+  included do
+    attr_accessor :trace_artifacts
+
+    set_callback :page_ready, :before, :start_trace
+    set_callback :page_close, :before, :stop_trace
+  end
+
+  private
+    def trace_dir
+      Upright.configuration.recording_base_dir
+    end
+
+    def start_trace
+      # Authenticated probes are not traced at all: a trace records network
+      # activity, which for an authenticated context includes Cookie and
+      # Authorization headers. Unauthenticated probes carry no credentials, so
+      # they still get a screenshots-only trace with snapshots (DOM + network
+      # payloads) disabled.
+      return if authentication_service.present?
+
+      context.tracing.start(screenshots: true, snapshots: false)
+    end
+
+    def stop_trace
+      return if authentication_service.present?
+
+      trace_path = trace_dir.join("#{SecureRandom.hex}.zip").to_s
+      FileUtils.mkdir_p(trace_dir)
+      context.tracing.stop(path: trace_path)
+
+      self.trace_artifacts ||= []
+      trace_artifacts << { label: current_recording_label, path: trace_path }
+    rescue => error
+      Rails.error.report(error)
+    end
+
+    def attach_trace(probe_result)
+      Array(trace_artifacts).each do |artifact|
+        next unless File.exist?(artifact.fetch(:path))
+
+        File.open(artifact.fetch(:path), "rb") do |file|
+          Upright::Artifact.new(name: recording_artifact_filename(artifact[:label], "zip"), content: file).attach_to(probe_result, timestamped: true)
+        end
+
+        FileUtils.rm(artifact.fetch(:path))
+      end
+
+      self.trace_artifacts = []
+    end
+end

data/app/models/concerns/upright/playwright/video_recording.rb

@@ -4,7 +4,7 @@ module Upright::Playwright::VideoRecording
   VIDEO_SIZE = { width: 1280, height: 720 }
 
   included do
-    attr_accessor :video_path, :video_object
+    attr_accessor :video_artifacts, :pending_video_recording
 
     set_callback :page_ready, :after, :capture_video_reference
     set_callback :page_close, :after, :finalize_video
@@ -33,28 +33,69 @@ module Upright::Playwright::VideoRecording
     end
 
     def capture_video_reference
-      self.video_object = page.video if record_video?
+      return unless record_video?
+      return unless (video = page.video)
+
+      self.pending_video_recording = { label: current_recording_label, video: video }
     end
 
     def save_video
-      if video_object
-        self.video_path = video_dir.join("#{SecureRandom.hex}.webm").to_s
-        video_object.save_as(video_path)
-      end
+      return unless pending_video_recording
+
+      video_path = video_dir.join("#{SecureRandom.hex}.webm").to_s
+      pending_video_recording.fetch(:video).save_as(video_path)
+
+      self.video_artifacts ||= []
+      video_artifacts << pending_video_recording.merge(path: video_path)
+    ensure
+      delete_source_video
+      self.pending_video_recording = nil
+    end
+
+    # Playwright writes its own source file per page; delete it after saving
+    # our copy so page@*.webm recordings don't accumulate on disk.
+    def delete_source_video
+      pending_video_recording&.fetch(:video)&.delete
+    rescue => error
+      Rails.error.report(error)
     end
 
     def attach_video(probe_result)
-      if video_path
-        File.open(video_path, "rb") do |file|
-          Upright::Artifact.new(name: "#{probe_name}.webm", content: file).attach_to(probe_result, timestamped: true)
-        end
+      Array(video_artifacts).each do |artifact|
+        next unless File.exist?(artifact.fetch(:path))
 
-        if logger.respond_to?(:struct) && probe_result.artifacts.any?
-          logger.struct probe_artifact_url: Rails.application.routes.url_helpers.rails_blob_url(probe_result.artifacts.first, expires_in: 24.hours)
+        File.open(artifact.fetch(:path), "rb") do |file|
+          Upright::Artifact.new(name: recording_artifact_filename(artifact[:label], "webm"), content: file).attach_to(probe_result, timestamped: true)
         end
 
-        FileUtils.rm(video_path)
-        self.video_path = nil
+        FileUtils.rm(artifact.fetch(:path))
+      end
+
+      if logger.respond_to?(:struct)
+        video_artifact = probe_result.artifacts.find { |attached| attached.content_type == "video/webm" }
+        if video_artifact
+          # Log the blob key, never a signed URL: signed URLs in logs/OTEL are
+          # bearer credentials for the recording.
+          logger.struct probe_artifact_key: video_artifact.key
+        end
       end
+
+      self.video_artifacts = []
+    end
+
+    def recording_artifact_filename(label, extension)
+      [ recording_artifact_basename(label), extension ].join(".")
+    end
+
+    def recording_artifact_basename(label)
+      [ artifact_recording_name, label ].compact.join(" ")
+    end
+
+    def artifact_recording_name
+      respond_to?(:probe_name) ? probe_name : service_name.to_s
+    end
+
+    def current_recording_label
+      nil
     end
 end

data/app/models/concerns/upright/probe_result/stale_cleanup.rb

@@ -8,7 +8,7 @@ module Upright::ProbeResult::StaleCleanup
     end
 
     def cleanup_stale_successes
-      ok.where(created_at: ...Upright.config.stale_success_threshold.ago).in_batches.destroy_all
+      ok.where(created_at: ...Upright.config.stale_success_threshold.ago).purge_in_batches
     end
 
     def cleanup_stale_failures
@@ -17,7 +17,31 @@ module Upright::ProbeResult::StaleCleanup
         fail.order(created_at: :desc).offset(Upright.config.failure_retention_limit).pick(:created_at)
       ].compact.max
 
-      fail.where(created_at: ..cutoff).in_batches.destroy_all
+      fail.where(created_at: ..cutoff).purge_in_batches
+    end
+
+    # Delete records and their artifacts without going through destroy_all's
+    # per-record callback cascade. has_many_attached's default dependent is
+    # :purge_later, which enqueues one ActiveStorage::PurgeJob per attachment
+    # — at ~13k attachments per hourly run on a busy site, that floods the
+    # queue DB enough to contend with probe writes. Here we issue batched
+    # DELETEs for the records and their attachment rows, then purge blobs
+    # inline so no jobs are enqueued.
+    def purge_in_batches
+      in_batches do |batch|
+        batch_ids = batch.pluck(:id)
+        attachments = ActiveStorage::Attachment.where(record_type: name, record_id: batch_ids)
+        blob_ids = attachments.distinct.pluck(:blob_id)
+
+        attachments.delete_all
+        batch.delete_all
+
+        ActiveStorage::Blob.unattached.where(id: blob_ids).find_each do |blob|
+          blob.purge
+        rescue => e
+          Rails.error.report(e)
+        end
+      end
     end
   end
 end

data/app/models/concerns/upright/probeable.rb

@@ -4,12 +4,16 @@ module Upright::Probeable
 
   ALERT_SEVERITIES = %i[ medium high critical ]
 
+  mattr_accessor :probe_classes, default: []
+
   included do
     attr_writer :logger
 
     def logger
       @logger || Rails.logger
     end
+
+    Upright::Probeable.probe_classes |= [ self ]
   end
 
   class_methods do
@@ -61,7 +65,11 @@ module Upright::Probeable
   end
 
   def probe_service
-    nil
+    try(:service)
+  end
+
+  def probe_key
+    [ probe_name, probe_type, probe_target ]
   end
 
   def probe_alert_severity

data/app/models/concerns/upright/services/live_status.rb

@@ -0,0 +1,69 @@
+module Upright::Services::LiveStatus
+  extend ActiveSupport::Concern
+
+  OUTAGE_LOOKBACK = 24.hours
+
+  # Matches the public status page's `expires_in`, so a burst of anonymous
+  # hits costs at most one Prometheus round trip per service per window
+  # instead of amplifying every request into live queries.
+  CACHE_TTL = 15.seconds
+
+  def live_status
+    Upright::Status.for(live_up_fraction)
+  end
+
+  # Earliest moment of the current outage, or nil if the service is currently
+  # clear OR the outage predates OUTAGE_LOOKBACK. Callers should treat nil on a
+  # non-operational service as "longer than the live window."
+  def current_outage_started_at(now: Time.current)
+    history = live_down_history(now: now)
+    last_clear = history.rindex { |_ts, value| value.to_f == 0 }
+
+    if last_clear && last_clear < history.length - 1
+      Time.zone.at(history[last_clear + 1].first.to_f)
+    end
+  end
+
+  private
+    def live_up_fraction
+      1 - live_down_fraction
+    end
+
+    def live_down_fraction
+      cached :live_down_fraction do
+        response = Upright.prometheus_client.query(
+          query: live_down_query
+        ).deep_symbolize_keys
+        response.dig(:result, 0, :value, 1).to_f
+      end
+    end
+
+    # Cached without regard to `now:` — a window up to CACHE_TTL stale is
+    # within the freshness the status page already advertises.
+    def live_down_history(now:)
+      cached :live_down_history do
+        response = Upright.prometheus_client.query_range(
+          query: live_down_query,
+          start: (now - OUTAGE_LOOKBACK).iso8601,
+          end:   now.iso8601,
+          step:  "300s"
+        ).deep_symbolize_keys
+        response.dig(:result, 0, :values) || []
+      end
+    end
+
+    def cached(name, &block)
+      Rails.cache.fetch([ "upright", name, code ], expires_in: CACHE_TTL, &block)
+    end
+
+    def live_down_query
+      matchers = [ %(probe_service="#{code}"), %(environment="#{Rails.env}"), %(type=~"#{uptime_probe_types_pattern}") ]
+      %(max(upright:probe_down_fraction{#{matchers.join(",")}}) or vector(0))
+    end
+
+    # Each type escaped as a regex literal, then the pattern escaped again for
+    # the PromQL string it sits in, so "api.v2" doesn't also match "apiXv2".
+    def uptime_probe_types_pattern
+      uptime_probe_types.map { |type| Regexp.escape(type) }.join("|").gsub(/["\\]/) { |char| "\\#{char}" }
+    end
+end

data/app/models/concerns/upright/services/maintenance_status.rb

@@ -0,0 +1,19 @@
+module Upright::Services::MaintenanceStatus
+  extend ActiveSupport::Concern
+
+  def active_maintenance
+    Upright::Maintenance.active.for_service(code).first
+  end
+
+  def maintenance_active?
+    active_maintenance.present?
+  end
+
+  def upcoming_maintenances
+    Upright::Maintenance.upcoming.for_service(code).order(:starts_at)
+  end
+
+  def active_incidents
+    Upright::Incident.reactive.active.for_service(code)
+  end
+end

data/app/models/upright/artifact.rb

@@ -6,7 +6,8 @@ class Upright::Artifact
     "html" => "download",
     "xml"  => "download",
     "txt"  => "download",
-    "bin"  => "download"
+    "bin"  => "download",
+    "zip"  => "download"
   }.freeze
 
   attr_reader :filename, :content

data/app/models/upright/http/request.rb

@@ -1,4 +1,9 @@
 class Upright::HTTP::Request
+  # Verbose curl output echoes request/response headers and credential lines.
+  # Redact their values before the log is attached as a viewable artifact.
+  SENSITIVE_HEADER = /^(\s*(?:authorization|proxy-authorization|cookie|set-cookie)\s*:\s*)[^\r\n]+/i
+  SENSITIVE_CREDENTIAL = /^(\s*[^\r\n:=]*(?:userpwd|password|credential)[^\r\n:=]*\s*[:=]\s*)[^\r\n]+/i
+
   attr_reader :url, :options
 
   def initialize(url, **options)
@@ -50,10 +55,14 @@ class Upright::HTTP::Request
     def build_log(response)
       StringIO.new.tap do |log|
         if response.debug_info
-          response.debug_info.text.each { |msg| log.puts "* #{msg.chomp}" }
-          response.debug_info.header_out.each { |msg| log.puts "> #{msg.chomp}" }
-          response.debug_info.header_in.each { |msg| log.puts "< #{msg.chomp}" }
+          response.debug_info.text.each { |msg| log.puts "* #{redact(msg.chomp)}" }
+          response.debug_info.header_out.each { |msg| log.puts "> #{redact(msg.chomp)}" }
+          response.debug_info.header_in.each { |msg| log.puts "< #{redact(msg.chomp)}" }
         end
       end
     end
+
+    def redact(message)
+      message.scrub.gsub(SENSITIVE_HEADER, '\1[REDACTED]').gsub(SENSITIVE_CREDENTIAL, '\1[REDACTED]')
+    end
 end

data/app/models/upright/incident/automatic_report.rb

@@ -0,0 +1,3 @@
+class Upright::Incident::AutomaticReport < Upright::PersistentRecord
+  belongs_to :incident, class_name: "Upright::Incident", inverse_of: :automatic_report, optional: true
+end

data/app/models/upright/incident.rb

@@ -0,0 +1,87 @@
+class Upright::Incident < Upright::PersistentRecord
+  include Upright::Incidents::Lifecycle
+  include Upright::Incidents::AutoReporting
+
+  attr_accessor :body
+
+  enum :status, %w[ investigating monitoring resolved
+                    scheduled in_progress completed ].index_with(&:itself)
+  enum :impact, %w[ minor major critical maintenance ].index_with(&:itself), prefix: true, default: "minor"
+
+  STATUSES          = %w[ investigating monitoring resolved ]
+  TERMINAL_STATUSES = %w[ resolved ]
+  IMPACTS           = %w[ minor major critical ]
+
+  IMPACT_STATUS = { "minor" => :degraded, "major" => :partial_outage, "critical" => :major_outage }
+
+  TEMPLATES = {
+    down: "%{services} %{is_or_are} down. We are investigating.",
+    investigating: "We are investigating the cause and will post updates here.",
+    back_up: "%{services} %{is_or_are} back up and operating normally."
+  }
+
+  def self.active_statuses
+    reactive.active.map { |incident| IMPACT_STATUS.fetch(incident.impact) }
+  end
+
+  has_many :updates, -> { order(created_at: :desc) }, class_name: "Upright::IncidentUpdate", inverse_of: :incident, dependent: :destroy
+  has_many :affected_services, class_name: "Upright::IncidentAffectedService", inverse_of: :incident, dependent: :destroy
+  has_one :automatic_report, class_name: "Upright::Incident::AutomaticReport", inverse_of: :incident, dependent: :destroy
+
+  # An incident is public-facing when it affects at least one public service.
+  # Everything else — internal-only or untagged — stays off the public status
+  # page entirely (fail closed).
+  scope :public_facing, -> { for_service(Upright::Service.public_facing.map(&:code)).distinct }
+
+  validates :title, :starts_at, presence: true
+  validates :status, inclusion: { in: ->(incident) { incident.class::STATUSES } }
+  validates :impact, inclusion: { in: ->(incident) { incident.class::IMPACTS } }
+
+  before_validation :set_default_status, on: :create
+  before_create { self.created_by ||= Upright::Current.user&.name }
+  before_update { self.updated_by = Upright::Current.user.name if Upright::Current.user }
+  after_create :record_initial_update
+
+  def maintenance? = false
+
+  def service_codes = affected_services.map(&:service_code)
+
+  def service_codes=(codes)
+    self.affected_services = Array(codes).reject(&:blank?).uniq.map do |code|
+      affected_services.find { |s| s.service_code == code } ||
+        affected_services.build(service_code: code)
+    end
+  end
+
+  def services
+    service_codes.filter_map { |code| Upright::Service.find_by(code: code) }
+  end
+
+  def public_services
+    services.select { |service| service[:public] }
+  end
+
+  def update_body_for(key)
+    subjects = services
+    template = subjects.filter_map { |service| service.incident_update_template(key) }.uniq
+    template = template.one? ? template.first : TEMPLATES.fetch(key)
+
+    template % {
+      services: subjects.any? ? subjects.map(&:name).to_sentence : "This service",
+      is_or_are: subjects.many? ? "are" : "is"
+    }
+  end
+
+  private
+    def set_default_status
+      self.status ||= self.class::STATUSES.first
+    end
+
+    def record_initial_update
+      updates.create!(status: status, body: body.presence || default_initial_body)
+    end
+
+    def default_initial_body
+      maintenance? ? "Maintenance scheduled." : "We are investigating."
+    end
+end

data/app/models/upright/incident_affected_service.rb

@@ -0,0 +1,9 @@
+class Upright::IncidentAffectedService < Upright::PersistentRecord
+  belongs_to :incident, class_name: "Upright::Incident", inverse_of: :affected_services
+
+  validates :service_code, presence: true,
+    uniqueness: { scope: :incident_id },
+    inclusion: { in: ->(_) { Upright::Service.all.map(&:code) } }
+
+  def service = Upright::Service.find_by(code: service_code)
+end

data/app/models/upright/incident_update.rb

@@ -0,0 +1,7 @@
+class Upright::IncidentUpdate < Upright::PersistentRecord
+  belongs_to :incident, class_name: "Upright::Incident", inverse_of: :updates
+
+  validates :status, presence: true
+
+  before_create { self.created_by ||= Upright::Current.user&.name || "System" }
+end