railwatch 0.1.4 → 0.2.0

data/app/jobs/railwatch/detect_anomalies_job.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+module Railwatch
+  # Compares each enabled anomaly rule's current window against the same clock
+  # window on each of the previous `baseline_days` days and opens an anomaly
+  # issue when the current value sits above the baseline's mean plus
+  # `deviation` standard deviations. Runs on a schedule via AnomalyScanJob.
+  class DetectAnomaliesJob < ApplicationJob
+    include DetectionSnapshotting
+
+    queue_as :default
+
+    TYPE_FOR = DetectPerformanceIssuesJob::TYPE_FOR
+    # Fewer baseline days than this and the standard deviation is noise.
+    MIN_BASELINE_DAYS = AnomalyRule::MIN_BASELINE_DAYS
+    # Volume metrics on a handful of events swing wildly; ignore them.
+    MIN_EVENTS = 20
+    COOL_DOWN = 60.minutes
+    MAX_GROUPS_PER_RULE = 50
+    MAX_CURRENT_ROWS_PER_RULE = MAX_GROUPS_PER_RULE * 25
+    MAX_BASELINE_ROWS_PER_RULE = 50_000
+
+    def perform(environment)
+      now = Time.current
+      environment.anomaly_rules.enabled.find_each do |rule|
+        detect(environment, rule, now).each { |detection| open_issue(environment, rule, now, detection) }
+      end
+    end
+
+    private
+
+    def detect(environment, rule, now)
+      from = now - rule.window_minutes.minutes
+      type = TYPE_FOR.fetch(rule.target_kind)
+      environment.with_telemetry do
+        groups = current_groups(rule, type, from, now)
+        baseline = baseline_rows(rule, type, groups.keys, from, now)
+        return [] unless baseline
+
+        groups.filter_map do |group_hash, (name, summary)|
+          anomaly_for(rule, group_hash, name, summary, baseline.fetch(group_hash, []), from, now)
+        end
+      end
+    end
+
+    # Rollups are hourly, so a window shorter than an hour reads the whole
+    # enclosing hour bucket (Rollup.between rounds `from` down); the baseline
+    # windows are shifted by whole days and land on the same hour, so the
+    # comparison stays like-for-like.
+    def current_groups(rule, type, from, to)
+      scope = Telemetry::Rollup.for_type(type).between(from, to)
+      scope = scope.where(name: rule.target) unless rule.target == "*"
+      group_hashes = scope.reorder(nil).group(:group_hash).order(Arel.sql("SUM(count) DESC"), :group_hash)
+                          .limit(MAX_GROUPS_PER_RULE + 1).pluck(:group_hash)
+      if group_hashes.length > MAX_GROUPS_PER_RULE
+        Rails.logger.warn("anomaly detector evaluated only the #{MAX_GROUPS_PER_RULE} busiest groups rule_id=#{rule.id}")
+        group_hashes.pop
+      end
+      rows = scope.where(group_hash: group_hashes).order(:bucket, :group_hash).limit(MAX_CURRENT_ROWS_PER_RULE + 1).to_a
+      if rows.length > MAX_CURRENT_ROWS_PER_RULE
+        Rails.logger.error("anomaly detector skipped rule_id=#{rule.id}: more than #{MAX_CURRENT_ROWS_PER_RULE} current rollups")
+        return {}
+      end
+
+      rows.group_by(&:group_hash)
+          .transform_values { |group_rows| [ group_rows.first.name, Telemetry::Rollup.summarize(group_rows) ] }
+    end
+
+    def anomaly_for(rule, group_hash, name, summary, baseline_rows, from, to)
+      return if cooling_down?(rule, group_hash)
+      current = value_for(rule.metric, summary, rule.window_minutes)
+      return if current.nil?
+      return if %w[count error_rate].include?(rule.metric) && summary[:count] < MIN_EVENTS
+
+      baseline = baseline_values(rule, baseline_rows, from, to)
+      return if baseline.size < MIN_BASELINE_DAYS
+      mean = baseline.sum / baseline.size
+      stddev = Math.sqrt(baseline.sum { |v| (v - mean)**2 } / baseline.size)
+      return unless current > mean + (rule.deviation * stddev) && current > mean * 1.25
+
+      # A perfectly flat baseline has no σ to measure against, so report the
+      # rule's own threshold rather than an infinite one.
+      sigmas = stddev.positive? ? (current - mean) / stddev : rule.deviation
+      { group_hash: group_hash, name: name, current: current.round(2), mean: mean.round(2),
+        stddev: stddev.round(2), sigmas: sigmas.round(2) }
+    end
+
+    # The same clock window on each of the previous `baseline_days` days. Days
+    # with no rollups at all are left out rather than counted as zero.
+    def baseline_values(rule, rows, from, to)
+      (1..rule.baseline_days).filter_map do |days|
+        day_rows = rows.select { |row| row.bucket.between?((from - days.days).beginning_of_hour, to - days.days) }
+        next if day_rows.empty?
+        value_for(rule.metric, Telemetry::Rollup.summarize(day_rows), rule.window_minutes)
+      end
+    end
+
+    # One query for every group's baseline, reading only the hour buckets the
+    # baseline windows actually cover rather than the whole `baseline_days` span.
+    def baseline_rows(rule, type, group_hashes, from, to)
+      return {} if group_hashes.empty?
+
+      rows = Telemetry::Rollup.for_type(type)
+               .where(group_hash: group_hashes, bucket: baseline_buckets(rule, from, to))
+               .order(:bucket, :group_hash).limit(MAX_BASELINE_ROWS_PER_RULE + 1).to_a
+      if rows.length > MAX_BASELINE_ROWS_PER_RULE
+        Rails.logger.error("anomaly detector skipped rule_id=#{rule.id}: more than #{MAX_BASELINE_ROWS_PER_RULE} baseline rollups")
+        return nil
+      end
+      rows.group_by(&:group_hash)
+    end
+
+    # The hour buckets covered by the same clock window on each of the previous
+    # `baseline_days` days.
+    def baseline_buckets(rule, from, to)
+      (1..rule.baseline_days).flat_map do |days|
+        bucket = (from - days.days).beginning_of_hour
+        last = to - days.days
+        buckets = []
+        while bucket <= last
+          buckets << bucket
+          bucket += 1.hour
+        end
+        buckets
+      end
+    end
+
+    def value_for(metric, summary, window_minutes)
+      case metric
+      when "p95" then summary[:p95] / 1000.0
+      when "avg" then summary[:avg] / 1000.0
+      when "count" then summary[:count] / window_minutes.to_f
+      when "error_rate" then summary[:count].zero? ? nil : (summary[:errors] * 100.0 / summary[:count])
+      end
+    end
+
+    # Reads the primary database: one open anomaly issue per (rule, group) is
+    # enough for an hour, however often the scan runs.
+    def cooling_down?(rule, group_hash)
+      Issue.where(environment_id: rule.environment_id, group_hash: issue_group_hash(rule, group_hash))
+           .where(last_seen_at: COOL_DOWN.ago..).exists?
+    end
+
+    def issue_group_hash(rule, group_hash)
+      "anomaly:#{rule.id}:#{group_hash}"
+    end
+
+    def open_issue(environment, rule, now, detection)
+      issue, outcome = Issue.record_occurrence!(
+        environment: environment, group_hash: issue_group_hash(rule, detection[:group_hash]), kind: "anomaly",
+        title: "#{rule.metric} of #{detection[:name]} is #{detection[:sigmas].round(1)}σ above its #{rule.baseline_days}-day baseline (#{detection[:current]} vs #{detection[:mean]})",
+        culprit: detection[:name], occurred_at: now, deploy: nil, user_ref: nil,
+        sample: { rule_id: rule.id, metric: rule.metric, current: detection[:current], mean: detection[:mean],
+                  stddev: detection[:stddev], sigmas: detection[:sigmas], window_minutes: rule.window_minutes })
+      carry_detection(issue, outcome) do
+        IssueDetectionSnapshot.capture(
+          environment: environment, kind: "anomaly", telemetry_type: TYPE_FOR.fetch(rule.target_kind),
+          telemetry_group_hash: detection[:group_hash], target: detection[:name], metric: rule.metric,
+          from: now - rule.window_minutes.minutes, to: now, value: detection[:current],
+          baseline: detection.slice(:mean, :stddev, :sigmas).merge(days: rule.baseline_days, deviation: rule.deviation),
+          rule: { type: "anomaly", id: rule.id, target_kind: rule.target_kind, target: rule.target }
+        )
+      end
+      if outcome == :new || outcome == :regressed
+        issue.fire_alerts!("anomaly", rule_id: rule.id, metric: rule.metric, current: detection[:current],
+          mean: detection[:mean], sigmas: detection[:sigmas])
+      end
+      rule.update_columns(last_fired_at: now)
+    end
+  end
+end

data/app/jobs/railwatch/detect_performance_issues_job.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Railwatch
+  # Evaluates every threshold for an environment against the last window of
+  # rollups and opens performance issues. Runs on a schedule (recurring.yml).
+  class DetectPerformanceIssuesJob < ApplicationJob
+    include DetectionSnapshotting
+
+    queue_as :default
+
+    MAX_GROUPS_PER_RULE = 200
+    MAX_ROLLUP_ROWS_PER_RULE = MAX_GROUPS_PER_RULE * 25
+
+    TYPE_FOR = { "requests" => "request", "jobs" => "job_attempt", "commands" => "command", "queries" => "query",
+                 "scheduled_tasks" => "scheduled_task", "outgoing_requests" => "outgoing_request" }.freeze
+
+    def perform(environment)
+      now = Time.current
+      environment.thresholds.find_each do |threshold|
+        from = now - threshold.window_minutes.minutes
+        type = TYPE_FOR.fetch(threshold.target_kind)
+        groups = environment.with_telemetry do
+          scope = Telemetry::Rollup.for_type(type).between(from, now)
+          scope = scope.where(name: threshold.target) unless threshold.target == "*"
+          bounded_groups(scope, threshold)
+        end
+        groups.each do |group_hash, (name, summary)|
+          value = value_for(threshold.metric, summary)
+          next if value.nil? || value <= threshold.limit
+          open_issue(environment, threshold, group_hash, name, value, from, now)
+        end
+      end
+    end
+
+    private
+
+    def bounded_groups(scope, threshold)
+      group_hashes = scope.reorder(nil).group(:group_hash).order(Arel.sql("SUM(count) DESC"), :group_hash)
+                          .limit(MAX_GROUPS_PER_RULE + 1).pluck(:group_hash)
+      if group_hashes.length > MAX_GROUPS_PER_RULE
+        Rails.logger.warn("threshold detector evaluated only the #{MAX_GROUPS_PER_RULE} busiest groups threshold_id=#{threshold.id}")
+        group_hashes.pop
+      end
+      rows = scope.where(group_hash: group_hashes).order(:bucket, :group_hash).limit(MAX_ROLLUP_ROWS_PER_RULE + 1).to_a
+      if rows.length > MAX_ROLLUP_ROWS_PER_RULE
+        Rails.logger.error("threshold detector skipped threshold_id=#{threshold.id}: more than #{MAX_ROLLUP_ROWS_PER_RULE} rollups")
+        return {}
+      end
+
+      rows.group_by(&:group_hash)
+          .transform_values { |group_rows| [ group_rows.first.name, Telemetry::Rollup.summarize(group_rows) ] }
+    end
+
+    def value_for(metric, s)
+      case metric
+      when "p95" then s[:p95] / 1000.0
+      when "max" then s[:max] / 1000.0
+      when "avg" then s[:avg] / 1000.0
+      when "error_rate", "failure_rate" then s[:count].zero? ? nil : (s[:errors] * 100.0 / s[:count])
+      end
+    end
+
+    def open_issue(environment, threshold, group_hash, name, value, from, now)
+      issue, outcome = Issue.record_occurrence!(
+        environment: environment, group_hash: "perf:#{threshold.id}:#{group_hash}", kind: "performance",
+        title: "#{name} exceeded #{threshold.metric} #{threshold.limit}#{threshold.metric.end_with?('rate') ? '%' : 'ms'} (#{value.round(1)})",
+        culprit: name, occurred_at: now, deploy: nil, user_ref: nil,
+        sample: { threshold_id: threshold.id, value: value.round(2), metric: threshold.metric, limit: threshold.limit })
+      carry_detection(issue, outcome) do
+        IssueDetectionSnapshot.capture(
+          environment: environment, kind: "performance", telemetry_type: TYPE_FOR.fetch(threshold.target_kind),
+          telemetry_group_hash: group_hash, target: name, metric: threshold.metric, from: from, to: now,
+          value: value.round(2), limit: threshold.limit,
+          rule: { type: "threshold", id: threshold.id, target_kind: threshold.target_kind, target: threshold.target }
+        )
+      end
+      return unless outcome == :new || outcome == :regressed
+      payload = { issue_key: issue.key, title: issue.title, environment: environment.name }
+      environment.application.alert_rules.where(event: "threshold").find_each do |rule|
+        next unless rule.matches?(issue: issue, payload: payload)
+        rule.fire!(event: "threshold", issue: issue, payload: payload)
+      end
+    end
+  end
+end

data/app/jobs/railwatch/group_exceptions_job.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Railwatch
+  # Turns newly ingested exception rows into Issues (create or bump), and
+  # fires new / regressed alerts.
+  class GroupExceptionsJob < ApplicationJob
+    queue_as :default
+
+    # A browser whose in-flight request died while kamal-proxy swapped
+    # containers reports a network error a few seconds after the deploy
+    # marker. That is the deploy working, not the app failing, and it opened
+    # (then regressed) an issue on every deploy today. The exception rows are
+    # still recorded; they just do not become an issue when they land inside
+    # this window around a deploy. Same idea as CheckScheduledTasksJob's
+    # DEPLOY_GRACE for missed runs.
+    DEPLOY_GRACE = 3.minutes
+    DEPLOY_NETWORK_ERRORS = %w[HttpNetworkError AxiosError InertiaException NetworkError TypeError:NetworkError].freeze
+
+    # batch_id: the embedded ledger's id for the batch these rows came from.
+    # With one, each group's occurrence count is committed together with a
+    # FollowupReceipt for (batch, group), so running this again for the same
+    # batch (a crash after the count but before the ledger was cleared, or
+    # the inline drain racing the maintenance drain) counts nothing twice.
+    # The hosted platform enqueues this once per HTTP batch and passes none.
+    def perform(environment, exception_ids, batch_id: nil)
+      rows = environment.with_telemetry { Telemetry::Exception.where(id: exception_ids).to_a }
+      rows = rows.reject { |row| deploy_swap_noise?(environment, row) }
+      rows.group_by(&:group_hash).each do |group_hash, group|
+        issue = outcome = nil
+        Issue.transaction do
+          next if batch_id && !FollowupReceipt.claim!(batch_id: batch_id, group_hash: group_hash)
+
+          latest = group.max_by(&:occurred_at)
+          issue, outcome = Issue.record_occurrence!(
+            environment: environment, group_hash: group_hash, kind: "exception",
+            title: "#{latest.class_name}: #{latest.message.to_s.first(200)}",
+            culprit: [ latest.file, latest.line ].compact.join(":").presence,
+            occurred_at: latest.occurred_at, deploy: latest.deploy, user_ref: latest.user_ref, source: latest.source,
+            sample: { exception_id: latest.id, handled: latest.handled, execution_id: latest.execution_id,
+                      execution_preview: latest.execution_preview, deploy: latest.deploy,
+                      fingerprint: latest.fingerprint, fingerprint_source: latest.fingerprint_source })
+          issue.increment!(:occurrences, group.size - 1) if group.size > 1
+        end
+        next unless issue
+
+        update_affected_users(environment, issue) if affected_users_due?(issue, outcome)
+        alert(issue, outcome)
+      end
+    end
+
+    # The affected-user count is a DISTINCT over every retained occurrence of
+    # the group, so on a busy issue it grows with the retention window and
+    # used to run once per batch per group. Once per window per issue, and
+    # always for a new issue, keeps the number fresh at a bounded cost.
+    AFFECTED_USERS_WINDOW = 5.minutes
+    AFFECTED_USERS_AT = Concurrent::Map.new
+
+    private
+
+    def affected_users_due?(issue, outcome)
+      now = Time.current
+      return AFFECTED_USERS_AT[issue.id] = now if outcome == :new
+
+      last = AFFECTED_USERS_AT[issue.id]
+      return false if last && now - last < AFFECTED_USERS_WINDOW
+
+      AFFECTED_USERS_AT[issue.id] = now
+    end
+
+    def deploy_swap_noise?(environment, row)
+      return false unless row.source == "browser" && DEPLOY_NETWORK_ERRORS.include?(row.class_name)
+
+      last_deploy_at = environment.deploys.maximum(:deployed_at) or return false
+      row.occurred_at.between?(last_deploy_at - DEPLOY_GRACE, last_deploy_at + DEPLOY_GRACE)
+    end
+
+    def update_affected_users(environment, issue)
+      count = environment.with_telemetry { Telemetry::Exception.where(group_hash: issue.group_hash).where.not(user_ref: nil).distinct.count(:user_ref) }
+      issue.update_columns(affected_users: count)
+    end
+
+    def alert(issue, outcome)
+      event = { new: "new_issue", regressed: "regressed_issue" }[outcome] or return
+      payload = { issue_key: issue.key, title: issue.title, environment: issue.environment.name }
+      issue.application.alert_rules.where(event: event).find_each do |rule|
+        next unless rule.matches?(issue: issue, payload: payload)
+        rule.fire!(event: event, issue: issue, payload: payload)
+      end
+    end
+  end
+end

data/app/jobs/railwatch/optimize_telemetry_job.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Railwatch
+  # Refreshes SQLite's planner statistics for each environment's telemetry
+  # database. Nothing else ever runs ANALYZE on these files, so the planner
+  # has been choosing index shapes blind on tables that grow by hundreds of
+  # thousands of rows a day. PRAGMA optimize only re-analyzes tables whose
+  # stats look stale, so it is cheap to run daily; analysis_limit bounds the
+  # rows it samples per index.
+  class OptimizeTelemetryJob < ApplicationJob
+    queue_as :maintenance
+
+    def perform(environment = nil)
+      return [ Environment.current ].each { |env| self.class.perform_later(env) } if environment.nil?
+
+      environment.with_telemetry do
+        connection = TelemetryRecord.connection
+        connection.execute("PRAGMA analysis_limit = 1000")
+        connection.execute("PRAGMA optimize")
+      end
+    end
+  end
+end

data/app/jobs/railwatch/performance_scan_job.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Railwatch
+  class PerformanceScanJob < ApplicationJob
+    queue_as :default
+
+    def perform
+      [ Environment.current ].each { |env| DetectPerformanceIssuesJob.perform_later(env) }
+    end
+  end
+end

data/app/jobs/railwatch/prune_telemetry_job.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Railwatch
+  # Deletes raw rows past the account's retention window. Rollups are kept
+  # for 13 months so long-range charts keep working after raw rows are gone.
+  # Telemetry::Session is monitored-application telemetry; dashboard login
+  # sessions live in the primary database and are deliberately not touched.
+  class PruneTelemetryJob < ApplicationJob
+    queue_as :maintenance
+
+    AGGREGATE_RETENTION = 13.months
+    # Sessions have never been pruned, so the first run of this job would walk
+    # every hour an environment has ever recorded. Delete a bounded number of
+    # hourly buckets per run and let the nightly schedule drain the backlog.
+    SESSION_HOURS_PER_RUN = 48
+    BATCH = 5_000
+    # Per raw table per run. A backlog past this (a retention change, a
+    # restore of an old file) drains over successive runs rather than in one
+    # that holds SQLite's write lock for as long as it takes. In the embedded
+    # writer that long hold would look like a wedge and end the process.
+    MAX_BATCHES_PER_TABLE = 40
+
+    RAW = [ Telemetry::Execution, Telemetry::Query, Telemetry::Exception, Telemetry::CacheEvent, Telemetry::Mail,
+            Telemetry::Broadcast, Telemetry::Notification, Telemetry::OutgoingRequest, Telemetry::StorageOp,
+            Telemetry::ViewRender, Telemetry::Log, Telemetry::EnqueuedJob, Telemetry::Transaction,
+            Telemetry::NPlusOne, Telemetry::Deprecation, Telemetry::Visit, Telemetry::Span,
+            Telemetry::LlmCall,
+            Telemetry::Profile, Telemetry::Attachment ].freeze
+
+    # checkpoint: the WAL checkpoint mode run at the end. TRUNCATE (the
+    # default, for a dedicated worker) hands the space back to the filesystem
+    # but blocks every reader and writer while it does; PASSIVE checkpoints
+    # what it can without waiting on anyone, which is what a prune running
+    # inside a Puma worker (Railwatch::Maintenance) must use.
+    def perform(environment = nil, checkpoint: "TRUNCATE")
+      return [ Environment.current ].each { |env| self.class.perform_later(env) } if environment.nil?
+
+      cutoff = environment.retention_days.days.ago
+      aggregate_cutoff = AGGREGATE_RETENTION.ago
+      environment.with_telemetry do
+        unindex_logs(cutoff) if Telemetry::Log.fts_available?
+        RAW.each { |klass| prune(klass, cutoff) }
+        # NOT EXISTS rather than NOT IN: queries.group_hash is nullable, and one
+        # NULL in a NOT IN subquery makes it match nothing. A few hundred shapes.
+        Telemetry::QueryShape.where("NOT EXISTS (SELECT 1 FROM queries WHERE queries.group_hash = query_shapes.group_hash)").delete_all
+        prune_sessions(cutoff)
+        Telemetry::Rollup.where(bucket: ...aggregate_cutoff).delete_all
+        Telemetry::ReleaseHealth.where(bucket: ...aggregate_cutoff).in_batches(of: 5_000).delete_all
+        Telemetry::Person.where(last_seen_at: ...cutoff).in_batches(of: 5_000).delete_all
+        Telemetry::IngestBatch.where(received_at: ...cutoff).delete_all
+        Telemetry::Process.where(booted_at: ...cutoff).delete_all
+        Telemetry::HealthSample.where(sampled_at: ...cutoff).delete_all
+        TelemetryRecord.connection.execute("PRAGMA wal_checkpoint(#{checkpoint})") if TelemetryRecord.connection.adapter_name =~ /sqlite/i
+      end
+    end
+
+    private
+
+    # in_batches walks a table by primary key: its probe is "WHERE occurred_at
+    # < ? ORDER BY id LIMIT n", which no index serves, so SQLite scanned the
+    # whole table by rowid to find nothing to delete -- 42 seconds on the
+    # platform's own 12M-row queries table, every night, for an empty result.
+    # Ordered by occurred_at the same probe is one seek on any index led by
+    # that column, and the first page of expired rows is exactly the oldest
+    # ones. Tables without such an index still scan, but they are the small
+    # ones.
+    def prune(klass, cutoff)
+      MAX_BATCHES_PER_TABLE.times do
+        # Ordered by (occurred_at, id), not occurred_at alone: rows that tie
+        # on the timestamp at the limit boundary would otherwise be a
+        # different set here than in unindex_logs above, which leaves stale
+        # FTS postings behind and withdraws postings for logs that are staying.
+        ids = klass.where(occurred_at: ...cutoff).order(:occurred_at, :id).limit(BATCH).pluck(:id)
+        break if ids.empty?
+        klass.where(id: ids).delete_all
+        break if ids.size < BATCH
+      end
+    end
+
+    # Sessions are the raw input to the hourly release_health aggregate, so they
+    # are deleted a whole hour at a time. Flooring the cutoff to the start of its
+    # hour leaves the partial hour straddling the boundary in place: deleting
+    # only its expired prefix would let the next rollup rebuild that hour from
+    # the surviving tail and silently shrink an already correct aggregate.
+    def prune_sessions(cutoff)
+      expired = Telemetry::Session.where(occurred_at: ...cutoff.utc.beginning_of_hour)
+      SESSION_HOURS_PER_RUN.times do
+        oldest = expired.minimum(:occurred_at)
+        break if oldest.nil?
+
+        hour = oldest.utc.beginning_of_hour
+        expired.where(occurred_at: hour...(hour + 1.hour)).in_batches(of: 5_000).delete_all
+      end
+    end
+
+    # logs_fts is an external-content index with no triggers, so the postings
+    # for a log line have to be withdrawn while its row (and message) is still
+    # there. Deleting the rows first would leave the index permanently out of
+    # sync -- searches would keep returning rowids that no longer exist.
+    # Bounded the same way as the rows it precedes: the postings for at most
+    # MAX_BATCHES_PER_TABLE * BATCH expired lines are withdrawn per run, which
+    # is exactly the set prune(Telemetry::Log) will delete this run.
+    def unindex_logs(cutoff)
+      Telemetry::Log.connection.execute(Telemetry::Log.sanitize_sql_array([
+        "INSERT INTO logs_fts(logs_fts, rowid, message) SELECT 'delete', id, message FROM logs " \
+        "WHERE occurred_at < ? ORDER BY occurred_at, id LIMIT ?", cutoff, MAX_BATCHES_PER_TABLE * BATCH
+      ]))
+    end
+  end
+end

data/app/jobs/railwatch/release_health_rollup_job.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Railwatch
+  # Recomputes one hourly release_health bucket from raw `sessions` rows.
+  # Idempotent, and debounced by the same Solid Queue concurrency control
+  # RollupJob uses: while one recompute of a bucket is queued or running,
+  # further enqueues for it are discarded rather than piling up.
+  #
+  # The gem re-sends a live session every flush interval, so the same
+  # session_id lands in a bucket many times. Each id collapses to the worst
+  # status it reached (crashed beats errored beats ok beats started) and its
+  # longest reported duration, which is what makes "sessions" a session count
+  # rather than a record count.
+  class ReleaseHealthRollupJob < ApplicationJob
+    queue_as :rollups
+    limits_concurrency to: 1, key: ->(environment, bucket) { "release_health:#{environment.id}:#{bucket.to_i}" }, duration: 10.minutes, on_conflict: :discard
+
+    RANKS = { "started" => 0, "ok" => 1, "errored" => 2, "crashed" => 3 }.freeze
+    CRASHED = RANKS["crashed"]
+    ERRORED = RANKS["errored"]
+
+    def perform(environment, bucket)
+      bucket = bucket.utc.beginning_of_hour
+      # PruneTelemetryJob deletes raw sessions in whole hours below this floor.
+      # A rollup enqueued before the prune can run after it, and rebuilding a
+      # pruned hour would replace a complete aggregate with an empty one.
+      return if bucket < environment.retention_days.days.ago.utc.beginning_of_hour
+
+      environment.with_telemetry do
+        rows = Telemetry::Session.where(occurred_at: bucket...(bucket + 1.hour))
+          .where.not(deploy: nil).pluck(:deploy, :session_id, :status, :duration, :user_ref)
+        aggregates = rows.group_by(&:first).map { |deploy, group| aggregate(deploy, bucket, collapse(group)) }
+        Telemetry::ReleaseHealth.transaction do
+          Telemetry::ReleaseHealth.where(bucket: bucket).delete_all
+          Telemetry::ReleaseHealth.insert_all(aggregates) if aggregates.any?
+        end
+      end
+    end
+
+    private
+
+    # session_id => the worst status, longest duration, and first user seen for
+    # it in this bucket.
+    def collapse(rows)
+      rows.each_with_object({}) do |(_deploy, session_id, status, duration, user_ref), sessions|
+        session = sessions[session_id] ||= { rank: 0, duration: nil, user: nil }
+        session[:rank] = [ session[:rank], RANKS.fetch(status.to_s, 0) ].max
+        session[:duration] = [ session[:duration] || 0, duration ].max if duration
+        session[:user] ||= user_ref
+      end
+    end
+
+    def aggregate(deploy, bucket, sessions)
+      durations = sessions.values.filter_map { |s| s[:duration] }
+      users = sessions.values.filter_map { |s| s[:user] }.uniq
+      crashed_users = sessions.values.select { |s| s[:rank] == CRASHED }.filter_map { |s| s[:user] }.uniq
+      { deploy: deploy, bucket: bucket, sessions: sessions.size,
+       sessions_errored: sessions.values.count { |s| s[:rank] == ERRORED },
+       sessions_crashed: sessions.values.count { |s| s[:rank] == CRASHED },
+       users: users.size, users_crashed: crashed_users.size,
+       duration_sum: durations.sum, duration_count: durations.size }
+    end
+  end
+end

data/app/jobs/railwatch/rollup_catchup_job.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Railwatch
+  # Recomputes the current and previous hour so rollups are never more than a
+  # schedule tick stale, whatever happened to the per-batch RollupJob enqueues
+  # (debounced in the web process, and their concurrency semaphore can outlive
+  # a worker restart). The platform iterates every active environment; an
+  # embedded install has one.
+  class RollupCatchupJob < ApplicationJob
+    queue_as :rollups
+
+    def perform
+      env = Environment.current
+      now = Time.current
+      [ now.beginning_of_hour, (now - 1.hour).beginning_of_hour ].each do |bucket|
+        RollupJob.perform_now(env, bucket)
+        ReleaseHealthRollupJob.perform_now(env, bucket)
+      end
+    end
+  end
+end