upright 0.2.0 → 0.4.0

data/app/jobs/upright/rollups/daily_aggregation_job.rb

@@ -0,0 +1,14 @@
+class Upright::Rollups::DailyAggregationJob < Upright::ApplicationJob
+  queue_as :default
+
+  # Aggregates daily rollups for completed days only — today is still in progress
+  # and is represented live by Service#live_status, so persisting a half-day
+  # rollup would just produce a stale value the rest of the day.
+  def perform(past: 7.days)
+    left_unwritten = (past.ago.to_date..Date.yesterday).flat_map do |day|
+      Upright::Rollups::ProbeRollup.rollup_day(day)
+    end
+
+    Yabeda.upright_rollup_skipped_probes.set({}, left_unwritten.size)
+  end
+end

data/app/models/concerns/upright/incidents/auto_reporting.rb

@@ -0,0 +1,93 @@
+module Upright::Incidents::AutoReporting
+  extend ActiveSupport::Concern
+
+  RESOLVE_DELAY = 5.minutes
+
+  included do
+    scope :auto_created, -> { where(auto_created: true) }
+  end
+
+  class_methods do
+    def report_downtime(now: Time.current)
+      outages = Upright::Service.degraded
+
+      outages.each { |outage| report_outage(outage, observed_at: now) }
+      observe_recoveries(excluding: outages.map(&:service), observed_at: now)
+    end
+
+    private
+      def report_outage(outage, observed_at:)
+        if incident = reactive.unresolved.for_service(outage.service.code).first
+          incident.observe_downtime(outage, at: observed_at)
+        else
+          open_automatically(outage, at: observed_at)
+        end
+      rescue ActiveRecord::RecordNotUnique
+        joins(:automatic_report)
+          .find_by!(upright_incident_automatic_reports: { service_code: outage.service.code })
+          .observe_downtime(outage, at: observed_at)
+      end
+
+      def open_automatically(outage, at:)
+        new(
+          title: "#{outage.service.name} outage",
+          impact: outage.impact,
+          starts_at: outage.started_at_or_lookback(observed_at: at),
+          last_seen_down_at: at,
+          created_by: "System"
+        ).tap do |incident|
+          incident.automatically_affect(outage.service)
+          incident.body = incident.update_body_for(:down)
+          incident.save!
+        end
+      end
+
+      def observe_recoveries(excluding:, observed_at:)
+        degraded_codes = excluding.map(&:code)
+
+        auto_created.reactive.unresolved.find_each do |incident|
+          incident.observe_recovery(at: observed_at) unless (incident.service_codes & degraded_codes).any?
+        end
+      end
+  end
+
+  def automatically_affect(service)
+    self.auto_created = true
+    self.service_codes = [ service.code ]
+    build_automatic_report(service_code: service.code)
+  end
+
+  def observe_downtime(outage, at:)
+    self.last_seen_down_at = at
+    self.recovery_started_at = nil
+    escalate_to(outage.impact)
+    save!
+  end
+
+  def observe_recovery(at:)
+    if services.any?(&:maintenance_active?)
+      pause_recovery
+    elsif recovery_started_at.nil?
+      update!(recovery_started_at: at)
+    elsif recovery_started_at <= at - RESOLVE_DELAY
+      resolve_automatically(at: at)
+    end
+  end
+
+  private
+    def escalate_to(new_impact)
+      self.impact = new_impact if self.class::IMPACTS.index(new_impact) > self.class::IMPACTS.index(impact)
+    end
+
+    def pause_recovery
+      update!(recovery_started_at: nil) if recovery_started_at?
+    end
+
+    def resolve_automatically(at:)
+      record_update({ status: "resolved", body: update_body_for(:back_up) }, recorded_at: at)
+    end
+
+    def after_recording_terminal_status
+      automatic_report&.destroy!
+    end
+end

data/app/models/concerns/upright/incidents/lifecycle.rb

@@ -0,0 +1,47 @@
+module Upright::Incidents::Lifecycle
+  extend ActiveSupport::Concern
+
+  included do
+    scope :resolved,   -> { where.not(resolved_at: nil) }
+    scope :unresolved, -> { where(resolved_at: nil) }
+
+    scope :active,   -> { unresolved.where(starts_at: ..Time.current) }
+    scope :upcoming, -> { unresolved.where(starts_at: Time.current..) }
+    scope :past,     -> { resolved.order(starts_at: :desc) }
+
+    scope :reactive, -> { where.not(type: "Upright::Maintenance").or(where(type: nil)) }
+    scope :planned,  -> { where(type: "Upright::Maintenance") }
+
+    scope :for_service, ->(code) {
+      joins(:affected_services).where(upright_incident_affected_services: { service_code: code })
+    }
+  end
+
+  def active?   = resolved_at.nil? && starts_at <= Time.current
+  def upcoming? = resolved_at.nil? && starts_at > Time.current
+  def past?     = resolved_at.present?
+
+  def record_update(attributes = nil, recorded_at: Time.current, **update_attributes)
+    attributes = attributes ? attributes.merge(update_attributes) : update_attributes
+
+    updates.build(attributes.merge(created_at: recorded_at)).tap do |update|
+      next unless update.valid?
+
+      self.status = update.status
+      if self.class::TERMINAL_STATUSES.include?(update.status)
+        self.resolved_at ||= recorded_at
+      end
+      next unless valid?
+
+      transaction do
+        update.save
+        save
+        after_recording_terminal_status if self.class::TERMINAL_STATUSES.include?(update.status)
+      end
+    end
+  end
+
+  private
+    def after_recording_terminal_status
+    end
+end

data/app/models/concerns/upright/playwright/form_authentication.rb

@@ -3,6 +3,7 @@ module Upright::Playwright::FormAuthentication
 
   included do
     class_attribute :authentication_service
+    set_callback :page_ready, :after, :ensure_authenticated
   end
 
   class_methods do
@@ -12,16 +13,13 @@ module Upright::Playwright::FormAuthentication
   end
 
   private
-    def authenticated_context(browser, context_options = {})
+    def ensure_authenticated
       if authentication_service
-        authenticator_for(authentication_service).new(browser, context_options).authenticated_context
+        authenticator_for(authentication_service).new.ensure_authenticated(context, page)
       end
     end
 
     def authenticator_for(service)
-      # First try the host app's authenticator, then fall back to engine's
       "::Playwright::Authenticator::#{service.to_s.camelize}".constantize
-    rescue NameError
-      "Upright::Playwright::Authenticator::#{service.to_s.camelize}".constantize
     end
 end

data/app/models/concerns/upright/playwright/lifecycle.rb

@@ -16,14 +16,8 @@ module Upright::Playwright::Lifecycle
 
   private
     def with_browser(&block)
-      if ENV["LOCAL_PLAYWRIGHT"]
-        ::Playwright.create(playwright_cli_executable_path: "./node_modules/.bin/playwright") do |playwright|
-          playwright.chromium.launch(headless: false, &block)
-        end
-      else
-        server_url = Upright.configuration.playwright_server_url ||
-                     Rails.application.config_for(:playwright).fetch(:server_url)
-        ::Playwright.connect_to_browser_server(server_url, &block)
+      ::Playwright.create(playwright_cli_executable_path: Upright.configuration.playwright_cli_path) do |playwright|
+        playwright.chromium.launch(headless: ENV.fetch("HEADLESS", "true") != "false", &block)
       end
     end
 
@@ -33,13 +27,14 @@ module Upright::Playwright::Lifecycle
       run_callbacks :page_ready
       yield
     ensure
-      # Rescue each step independently so a failed close doesn't prevent video capture
-      page&.close rescue Rails.error.report($!)
-      context&.close rescue Rails.error.report($!)
-      run_callbacks :page_close
+      run_callbacks :page_close do
+        # Rescue each step independently so one failed close does not block artifact capture
+        page&.close rescue Rails.error.report($!)
+        context&.close rescue Rails.error.report($!)
+      end
     end
 
     def create_context(browser, **options)
-      authenticated_context(browser, options) || browser.new_context(userAgent: user_agent, **options)
+      browser.new_context(userAgent: user_agent, **options)
     end
 end

data/app/models/concerns/upright/playwright/trace_recording.rb

@@ -0,0 +1,53 @@
+module Upright::Playwright::TraceRecording
+  extend ActiveSupport::Concern
+
+  included do
+    attr_accessor :trace_artifacts
+
+    set_callback :page_ready, :before, :start_trace
+    set_callback :page_close, :before, :stop_trace
+  end
+
+  private
+    def trace_dir
+      Upright.configuration.recording_base_dir
+    end
+
+    def start_trace
+      # Authenticated probes are not traced at all: a trace records network
+      # activity, which for an authenticated context includes Cookie and
+      # Authorization headers. Unauthenticated probes carry no credentials, so
+      # they still get a screenshots-only trace with snapshots (DOM + network
+      # payloads) disabled.
+      return if authentication_service.present?
+
+      context.tracing.start(screenshots: true, snapshots: false)
+    end
+
+    def stop_trace
+      return if authentication_service.present?
+
+      trace_path = trace_dir.join("#{SecureRandom.hex}.zip").to_s
+      FileUtils.mkdir_p(trace_dir)
+      context.tracing.stop(path: trace_path)
+
+      self.trace_artifacts ||= []
+      trace_artifacts << { label: current_recording_label, path: trace_path }
+    rescue => error
+      Rails.error.report(error)
+    end
+
+    def attach_trace(probe_result)
+      Array(trace_artifacts).each do |artifact|
+        next unless File.exist?(artifact.fetch(:path))
+
+        File.open(artifact.fetch(:path), "rb") do |file|
+          Upright::Artifact.new(name: recording_artifact_filename(artifact[:label], "zip"), content: file).attach_to(probe_result, timestamped: true)
+        end
+
+        FileUtils.rm(artifact.fetch(:path))
+      end
+
+      self.trace_artifacts = []
+    end
+end

data/app/models/concerns/upright/playwright/video_recording.rb

@@ -4,7 +4,7 @@ module Upright::Playwright::VideoRecording
   VIDEO_SIZE = { width: 1280, height: 720 }
 
   included do
-    attr_accessor :video_path, :video_object
+    attr_accessor :video_artifacts, :pending_video_recording
 
     set_callback :page_ready, :after, :capture_video_reference
     set_callback :page_close, :after, :finalize_video
@@ -33,28 +33,69 @@ module Upright::Playwright::VideoRecording
     end
 
     def capture_video_reference
-      self.video_object = page.video if record_video?
+      return unless record_video?
+      return unless (video = page.video)
+
+      self.pending_video_recording = { label: current_recording_label, video: video }
     end
 
     def save_video
-      if video_object
-        self.video_path = video_dir.join("#{SecureRandom.hex}.webm").to_s
-        video_object.save_as(video_path)
-      end
+      return unless pending_video_recording
+
+      video_path = video_dir.join("#{SecureRandom.hex}.webm").to_s
+      pending_video_recording.fetch(:video).save_as(video_path)
+
+      self.video_artifacts ||= []
+      video_artifacts << pending_video_recording.merge(path: video_path)
+    ensure
+      delete_source_video
+      self.pending_video_recording = nil
+    end
+
+    # Playwright writes its own source file per page; delete it after saving
+    # our copy so page@*.webm recordings don't accumulate on disk.
+    def delete_source_video
+      pending_video_recording&.fetch(:video)&.delete
+    rescue => error
+      Rails.error.report(error)
     end
 
     def attach_video(probe_result)
-      if video_path
-        File.open(video_path, "rb") do |file|
-          Upright::Artifact.new(name: "#{probe_name}.webm", content: file).attach_to(probe_result, timestamped: true)
-        end
+      Array(video_artifacts).each do |artifact|
+        next unless File.exist?(artifact.fetch(:path))
 
-        if logger.respond_to?(:struct) && probe_result.artifacts.any?
-          logger.struct probe_artifact_url: Rails.application.routes.url_helpers.rails_blob_url(probe_result.artifacts.first, expires_in: 24.hours)
+        File.open(artifact.fetch(:path), "rb") do |file|
+          Upright::Artifact.new(name: recording_artifact_filename(artifact[:label], "webm"), content: file).attach_to(probe_result, timestamped: true)
         end
 
-        FileUtils.rm(video_path)
-        self.video_path = nil
+        FileUtils.rm(artifact.fetch(:path))
+      end
+
+      if logger.respond_to?(:struct)
+        video_artifact = probe_result.artifacts.find { |attached| attached.content_type == "video/webm" }
+        if video_artifact
+          # Log the blob key, never a signed URL: signed URLs in logs/OTEL are
+          # bearer credentials for the recording.
+          logger.struct probe_artifact_key: video_artifact.key
+        end
       end
+
+      self.video_artifacts = []
+    end
+
+    def recording_artifact_filename(label, extension)
+      [ recording_artifact_basename(label), extension ].join(".")
+    end
+
+    def recording_artifact_basename(label)
+      [ artifact_recording_name, label ].compact.join(" ")
+    end
+
+    def artifact_recording_name
+      respond_to?(:probe_name) ? probe_name : service_name.to_s
+    end
+
+    def current_recording_label
+      nil
     end
 end

data/app/models/concerns/upright/probe_result/stale_cleanup.rb

@@ -0,0 +1,47 @@
+module Upright::ProbeResult::StaleCleanup
+  extend ActiveSupport::Concern
+
+  class_methods do
+    def cleanup_stale
+      cleanup_stale_successes
+      cleanup_stale_failures
+    end
+
+    def cleanup_stale_successes
+      ok.where(created_at: ...Upright.config.stale_success_threshold.ago).purge_in_batches
+    end
+
+    def cleanup_stale_failures
+      cutoff = [
+        Upright.config.stale_failure_threshold.ago,
+        fail.order(created_at: :desc).offset(Upright.config.failure_retention_limit).pick(:created_at)
+      ].compact.max
+
+      fail.where(created_at: ..cutoff).purge_in_batches
+    end
+
+    # Delete records and their artifacts without going through destroy_all's
+    # per-record callback cascade. has_many_attached's default dependent is
+    # :purge_later, which enqueues one ActiveStorage::PurgeJob per attachment
+    # — at ~13k attachments per hourly run on a busy site, that floods the
+    # queue DB enough to contend with probe writes. Here we issue batched
+    # DELETEs for the records and their attachment rows, then purge blobs
+    # inline so no jobs are enqueued.
+    def purge_in_batches
+      in_batches do |batch|
+        batch_ids = batch.pluck(:id)
+        attachments = ActiveStorage::Attachment.where(record_type: name, record_id: batch_ids)
+        blob_ids = attachments.distinct.pluck(:blob_id)
+
+        attachments.delete_all
+        batch.delete_all
+
+        ActiveStorage::Blob.unattached.where(id: blob_ids).find_each do |blob|
+          blob.purge
+        rescue => e
+          Rails.error.report(e)
+        end
+      end
+    end
+  end
+end

data/app/models/concerns/upright/probeable.rb

@@ -2,7 +2,9 @@ module Upright::Probeable
   extend ActiveSupport::Concern
   include Upright::Staggerable
 
-  TYPES = %w[ http playwright smtp traceroute ]
+  ALERT_SEVERITIES = %i[ medium high critical ]
+
+  mattr_accessor :probe_classes, default: []
 
   included do
     attr_writer :logger
@@ -10,6 +12,8 @@ module Upright::Probeable
     def logger
       @logger || Rails.logger
     end
+
+    Upright::Probeable.probe_classes |= [ self ]
   end
 
   class_methods do
@@ -35,6 +39,7 @@ module Upright::Probeable
       probe_name: probe_name,
       probe_target: probe_target,
       probe_service: probe_service,
+      probe_alert_severity: probe_alert_severity,
       status: result[:status],
       duration: result[:duration],
       error: result[:error]
@@ -60,7 +65,16 @@ module Upright::Probeable
   end
 
   def probe_service
-    nil
+    try(:service)
+  end
+
+  def probe_key
+    [ probe_name, probe_type, probe_target ]
+  end
+
+  def probe_alert_severity
+    severity = try(:alert_severity)&.to_sym
+    ALERT_SEVERITIES.include?(severity) ? severity : :high
   end
 
   private

data/app/models/concerns/upright/services/live_status.rb

@@ -0,0 +1,69 @@
+module Upright::Services::LiveStatus
+  extend ActiveSupport::Concern
+
+  OUTAGE_LOOKBACK = 24.hours
+
+  # Matches the public status page's `expires_in`, so a burst of anonymous
+  # hits costs at most one Prometheus round trip per service per window
+  # instead of amplifying every request into live queries.
+  CACHE_TTL = 15.seconds
+
+  def live_status
+    Upright::Status.for(live_up_fraction)
+  end
+
+  # Earliest moment of the current outage, or nil if the service is currently
+  # clear OR the outage predates OUTAGE_LOOKBACK. Callers should treat nil on a
+  # non-operational service as "longer than the live window."
+  def current_outage_started_at(now: Time.current)
+    history = live_down_history(now: now)
+    last_clear = history.rindex { |_ts, value| value.to_f == 0 }
+
+    if last_clear && last_clear < history.length - 1
+      Time.zone.at(history[last_clear + 1].first.to_f)
+    end
+  end
+
+  private
+    def live_up_fraction
+      1 - live_down_fraction
+    end
+
+    def live_down_fraction
+      cached :live_down_fraction do
+        response = Upright.prometheus_client.query(
+          query: live_down_query
+        ).deep_symbolize_keys
+        response.dig(:result, 0, :value, 1).to_f
+      end
+    end
+
+    # Cached without regard to `now:` — a window up to CACHE_TTL stale is
+    # within the freshness the status page already advertises.
+    def live_down_history(now:)
+      cached :live_down_history do
+        response = Upright.prometheus_client.query_range(
+          query: live_down_query,
+          start: (now - OUTAGE_LOOKBACK).iso8601,
+          end:   now.iso8601,
+          step:  "300s"
+        ).deep_symbolize_keys
+        response.dig(:result, 0, :values) || []
+      end
+    end
+
+    def cached(name, &block)
+      Rails.cache.fetch([ "upright", name, code ], expires_in: CACHE_TTL, &block)
+    end
+
+    def live_down_query
+      matchers = [ %(probe_service="#{code}"), %(environment="#{Rails.env}"), %(type=~"#{uptime_probe_types_pattern}") ]
+      %(max(upright:probe_down_fraction{#{matchers.join(",")}}) or vector(0))
+    end
+
+    # Each type escaped as a regex literal, then the pattern escaped again for
+    # the PromQL string it sits in, so "api.v2" doesn't also match "apiXv2".
+    def uptime_probe_types_pattern
+      uptime_probe_types.map { |type| Regexp.escape(type) }.join("|").gsub(/["\\]/) { |char| "\\#{char}" }
+    end
+end

data/app/models/concerns/upright/services/maintenance_status.rb

@@ -0,0 +1,19 @@
+module Upright::Services::MaintenanceStatus
+  extend ActiveSupport::Concern
+
+  def active_maintenance
+    Upright::Maintenance.active.for_service(code).first
+  end
+
+  def maintenance_active?
+    active_maintenance.present?
+  end
+
+  def upcoming_maintenances
+    Upright::Maintenance.upcoming.for_service(code).order(:starts_at)
+  end
+
+  def active_incidents
+    Upright::Incident.reactive.active.for_service(code)
+  end
+end

data/app/models/upright/artifact.rb

@@ -6,7 +6,8 @@ class Upright::Artifact
     "html" => "download",
     "xml"  => "download",
     "txt"  => "download",
-    "bin"  => "download"
+    "bin"  => "download",
+    "zip"  => "download"
   }.freeze
 
   attr_reader :filename, :content

data/app/models/upright/http/request.rb

@@ -1,4 +1,9 @@
 class Upright::HTTP::Request
+  # Verbose curl output echoes request/response headers and credential lines.
+  # Redact their values before the log is attached as a viewable artifact.
+  SENSITIVE_HEADER = /^(\s*(?:authorization|proxy-authorization|cookie|set-cookie)\s*:\s*)[^\r\n]+/i
+  SENSITIVE_CREDENTIAL = /^(\s*[^\r\n:=]*(?:userpwd|password|credential)[^\r\n:=]*\s*[:=]\s*)[^\r\n]+/i
+
   attr_reader :url, :options
 
   def initialize(url, **options)
@@ -23,7 +28,7 @@ class Upright::HTTP::Request
         proxy: proxy_url,
         proxyuserpwd: proxy_userpwd,
         verbose: true,
-        forbid_reuse: true
+        forbid_reuse: proxy_url.nil?
       }.compact
     end
 
@@ -50,10 +55,14 @@ class Upright::HTTP::Request
     def build_log(response)
       StringIO.new.tap do |log|
         if response.debug_info
-          response.debug_info.text.each { |msg| log.puts "* #{msg.chomp}" }
-          response.debug_info.header_out.each { |msg| log.puts "> #{msg.chomp}" }
-          response.debug_info.header_in.each { |msg| log.puts "< #{msg.chomp}" }
+          response.debug_info.text.each { |msg| log.puts "* #{redact(msg.chomp)}" }
+          response.debug_info.header_out.each { |msg| log.puts "> #{redact(msg.chomp)}" }
+          response.debug_info.header_in.each { |msg| log.puts "< #{redact(msg.chomp)}" }
         end
       end
     end
+
+    def redact(message)
+      message.scrub.gsub(SENSITIVE_HEADER, '\1[REDACTED]').gsub(SENSITIVE_CREDENTIAL, '\1[REDACTED]')
+    end
 end

data/app/models/upright/incident/automatic_report.rb

@@ -0,0 +1,3 @@
+class Upright::Incident::AutomaticReport < Upright::PersistentRecord
+  belongs_to :incident, class_name: "Upright::Incident", inverse_of: :automatic_report, optional: true
+end