railwatch 0.9.3 → 1.0.0

data/app/jobs/railwatch/detect_performance_issues_job.rb

@@ -1,95 +0,0 @@
-# frozen_string_literal: true
-
-module Railwatch
-  # Evaluates every threshold for an environment against the last window of
-  # rollups and opens performance issues. Runs on a schedule (recurring.yml).
-  class DetectPerformanceIssuesJob < ApplicationJob
-    include DetectionSnapshotting
-
-    queue_as :default
-
-    MAX_GROUPS_PER_RULE = 200
-    MAX_ROLLUP_ROWS_PER_RULE = MAX_GROUPS_PER_RULE * 25
-
-    TYPE_FOR = { "requests" => "request", "jobs" => "job_attempt", "commands" => "command", "queries" => "query",
-                 "scheduled_tasks" => "scheduled_task", "outgoing_requests" => "outgoing_request",
-                 "llm_calls" => "llm_call", "llm_tools" => "llm_tool" }.freeze
-
-    def perform(environment)
-      now = Time.current
-      environment.thresholds.find_each do |threshold|
-        from = now - threshold.window_minutes.minutes
-        type = TYPE_FOR.fetch(threshold.target_kind)
-        groups = environment.with_telemetry do
-          scope = Telemetry::Rollup.for_type(type).between(from, now)
-          scope = scope.where(name: threshold.target) unless threshold.target == "*"
-          bounded_groups(scope, threshold)
-        end
-        groups.each do |group_hash, (name, summary)|
-          value = value_for(threshold.metric, summary)
-          next if value.nil? || value <= threshold.limit
-          open_issue(environment, threshold, group_hash, name, value, from, now)
-        end
-      end
-    end
-
-    private
-
-    def bounded_groups(scope, threshold)
-      group_hashes = scope.reorder(nil).group(:group_hash).order(Arel.sql("SUM(count) DESC"), :group_hash)
-                          .limit(MAX_GROUPS_PER_RULE + 1).pluck(:group_hash)
-      if group_hashes.length > MAX_GROUPS_PER_RULE
-        Rails.logger.warn("threshold detector evaluated only the #{MAX_GROUPS_PER_RULE} busiest groups threshold_id=#{threshold.id}")
-        group_hashes.pop
-      end
-      rows = scope.where(group_hash: group_hashes).order(:bucket, :group_hash).limit(MAX_ROLLUP_ROWS_PER_RULE + 1).to_a
-      if rows.length > MAX_ROLLUP_ROWS_PER_RULE
-        Rails.logger.error("threshold detector skipped threshold_id=#{threshold.id}: more than #{MAX_ROLLUP_ROWS_PER_RULE} rollups")
-        return {}
-      end
-
-      rows.group_by(&:group_hash)
-          .transform_values { |group_rows| [ group_rows.first.name, Telemetry::Rollup.summarize(group_rows) ] }
-    end
-
-    NANOS_PER_DOLLAR = 1_000_000_000.0
-
-    def value_for(metric, s)
-      extra = s[:extra] || {}
-      case metric
-      when "p95" then s[:p95] / 1000.0
-      when "max" then s[:max] / 1000.0
-      when "avg" then s[:avg] / 1000.0
-      when "error_rate", "failure_rate" then s[:count].zero? ? nil : (s[:errors] * 100.0 / s[:count])
-      # Money and tokens are sums over the window, not percentiles of it.
-      when "spend" then extra["cost_nanos"].to_i / NANOS_PER_DOLLAR
-      when "tokens" then extra["input_tokens"].to_i + extra["output_tokens"].to_i
-      # A cut-off answer is a successful call, so it is invisible to
-      # error_rate. This is the only way to be told about it.
-      when "truncation_rate" then s[:count].zero? ? nil : (extra["truncated"].to_i * 100.0 / s[:count])
-      end
-    end
-
-    def open_issue(environment, threshold, group_hash, name, value, from, now)
-      issue, outcome = Issue.record_occurrence!(
-        environment: environment, group_hash: "perf:#{threshold.id}:#{group_hash}", kind: "performance",
-        title: "#{name} exceeded #{threshold.metric} #{threshold.format_value(threshold.limit)} (#{threshold.format_value(value.round(2))})",
-        culprit: name, occurred_at: now, deploy: nil, user_ref: nil,
-        sample: { threshold_id: threshold.id, value: value.round(2), metric: threshold.metric, limit: threshold.limit })
-      carry_detection(issue, outcome) do
-        IssueDetectionSnapshot.capture(
-          environment: environment, kind: "performance", telemetry_type: TYPE_FOR.fetch(threshold.target_kind),
-          telemetry_group_hash: group_hash, target: name, metric: threshold.metric, from: from, to: now,
-          value: value.round(2), limit: threshold.limit,
-          rule: { type: "threshold", id: threshold.id, target_kind: threshold.target_kind, target: threshold.target }
-        )
-      end
-      return unless outcome == :new || outcome == :regressed
-      payload = { issue_key: issue.key, title: issue.title, environment: environment.name }
-      environment.application.alert_rules.where(event: "threshold").find_each do |rule|
-        next unless rule.matches?(issue: issue, payload: payload)
-        rule.fire!(event: "threshold", issue: issue, payload: payload)
-      end
-    end
-  end
-end

data/app/jobs/railwatch/group_exceptions_job.rb

@@ -1,91 +0,0 @@
-# frozen_string_literal: true
-
-module Railwatch
-  # Turns newly ingested exception rows into Issues (create or bump), and
-  # fires new / regressed alerts.
-  class GroupExceptionsJob < ApplicationJob
-    queue_as :default
-
-    # A browser whose in-flight request died while kamal-proxy swapped
-    # containers reports a network error a few seconds after the deploy
-    # marker. That is the deploy working, not the app failing, and it opened
-    # (then regressed) an issue on every deploy today. The exception rows are
-    # still recorded; they just do not become an issue when they land inside
-    # this window around a deploy. Same idea as CheckScheduledTasksJob's
-    # DEPLOY_GRACE for missed runs.
-    DEPLOY_GRACE = 3.minutes
-    DEPLOY_NETWORK_ERRORS = %w[HttpNetworkError AxiosError InertiaException NetworkError TypeError:NetworkError].freeze
-
-    # batch_id: the embedded ledger's id for the batch these rows came from.
-    # With one, each group's occurrence count is committed together with a
-    # FollowupReceipt for (batch, group), so running this again for the same
-    # batch (a crash after the count but before the ledger was cleared, or
-    # the inline drain racing the maintenance drain) counts nothing twice.
-    # The hosted platform enqueues this once per HTTP batch and passes none.
-    def perform(environment, exception_ids, batch_id: nil)
-      rows = environment.with_telemetry { Telemetry::Exception.where(id: exception_ids).to_a }
-      rows = rows.reject { |row| deploy_swap_noise?(environment, row) }
-      rows.group_by(&:group_hash).each do |group_hash, group|
-        issue = outcome = nil
-        Issue.transaction do
-          next if batch_id && !FollowupReceipt.claim!(batch_id: batch_id, group_hash: group_hash)
-
-          latest = group.max_by(&:occurred_at)
-          issue, outcome = Issue.record_occurrence!(
-            environment: environment, group_hash: group_hash, kind: "exception",
-            title: "#{latest.class_name}: #{latest.message.to_s.first(200)}",
-            culprit: [ latest.file, latest.line ].compact.join(":").presence,
-            occurred_at: latest.occurred_at, deploy: latest.deploy, user_ref: latest.user_ref, source: latest.source,
-            sample: { exception_id: latest.id, handled: latest.handled, execution_id: latest.execution_id,
-                      execution_preview: latest.execution_preview, deploy: latest.deploy,
-                      fingerprint: latest.fingerprint, fingerprint_source: latest.fingerprint_source })
-          issue.increment!(:occurrences, group.size - 1) if group.size > 1
-        end
-        next unless issue
-
-        update_affected_users(environment, issue) if affected_users_due?(issue, outcome)
-        alert(issue, outcome)
-      end
-    end
-
-    # The affected-user count is a DISTINCT over every retained occurrence of
-    # the group, so on a busy issue it grows with the retention window and
-    # used to run once per batch per group. Once per window per issue, and
-    # always for a new issue, keeps the number fresh at a bounded cost.
-    AFFECTED_USERS_WINDOW = 5.minutes
-    AFFECTED_USERS_AT = Concurrent::Map.new
-
-    private
-
-    def affected_users_due?(issue, outcome)
-      now = Time.current
-      return AFFECTED_USERS_AT[issue.id] = now if outcome == :new
-
-      last = AFFECTED_USERS_AT[issue.id]
-      return false if last && now - last < AFFECTED_USERS_WINDOW
-
-      AFFECTED_USERS_AT[issue.id] = now
-    end
-
-    def deploy_swap_noise?(environment, row)
-      return false unless row.source == "browser" && DEPLOY_NETWORK_ERRORS.include?(row.class_name)
-
-      last_deploy_at = environment.deploys.maximum(:deployed_at) or return false
-      row.occurred_at.between?(last_deploy_at - DEPLOY_GRACE, last_deploy_at + DEPLOY_GRACE)
-    end
-
-    def update_affected_users(environment, issue)
-      count = environment.with_telemetry { Telemetry::Exception.where(group_hash: issue.group_hash).where.not(user_ref: nil).distinct.count(:user_ref) }
-      issue.update_columns(affected_users: count)
-    end
-
-    def alert(issue, outcome)
-      event = { new: "new_issue", regressed: "regressed_issue" }[outcome] or return
-      payload = { issue_key: issue.key, title: issue.title, environment: issue.environment.name }
-      issue.application.alert_rules.where(event: event).find_each do |rule|
-        next unless rule.matches?(issue: issue, payload: payload)
-        rule.fire!(event: event, issue: issue, payload: payload)
-      end
-    end
-  end
-end

data/app/jobs/railwatch/optimize_telemetry_job.rb

@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-module Railwatch
-  # Refreshes SQLite's planner statistics for each environment's telemetry
-  # database. Nothing else ever runs ANALYZE on these files, so the planner
-  # has been choosing index shapes blind on tables that grow by hundreds of
-  # thousands of rows a day. PRAGMA optimize only re-analyzes tables whose
-  # stats look stale, so it is cheap to run daily; analysis_limit bounds the
-  # rows it samples per index.
-  class OptimizeTelemetryJob < ApplicationJob
-    queue_as :maintenance
-
-    def perform(environment = nil)
-      return [ Environment.current ].each { |env| self.class.perform_later(env) } if environment.nil?
-
-      environment.with_telemetry do
-        connection = TelemetryRecord.connection
-        connection.execute("PRAGMA analysis_limit = 1000")
-        connection.execute("PRAGMA optimize")
-      end
-    end
-  end
-end

data/app/jobs/railwatch/performance_scan_job.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-module Railwatch
-  class PerformanceScanJob < ApplicationJob
-    queue_as :default
-
-    def perform
-      [ Environment.current ].each { |env| DetectPerformanceIssuesJob.perform_later(env) }
-    end
-  end
-end

data/app/jobs/railwatch/prune_telemetry_job.rb

@@ -1,129 +0,0 @@
-# frozen_string_literal: true
-
-module Railwatch
-  # Deletes raw rows past the account's retention window. Rollups are kept
-  # for 13 months so long-range charts keep working after raw rows are gone.
-  # Telemetry::Session is monitored-application telemetry; dashboard login
-  # sessions live in the primary database and are deliberately not touched.
-  class PruneTelemetryJob < ApplicationJob
-    queue_as :maintenance
-
-    AGGREGATE_RETENTION = 13.months
-    # Sessions have never been pruned, so the first run of this job would walk
-    # every hour an environment has ever recorded. Delete a bounded number of
-    # hourly buckets per run and let the nightly schedule drain the backlog.
-    SESSION_HOURS_PER_RUN = 48
-    BATCH = 5_000
-    # Per raw table per run. A backlog past this (a retention change, a
-    # restore of an old file) drains over successive runs rather than in one
-    # that holds SQLite's write lock for as long as it takes. In the embedded
-    # writer that long hold would look like a wedge and end the process.
-    MAX_BATCHES_PER_TABLE = 40
-
-    # Reclaim, bounded the same way the deletes are. Deleting rows only moves
-    # their pages to the freelist; PRAGMA incremental_vacuum is what hands
-    # them back to the filesystem, and it is only possible at all on a
-    # database in auto_vacuum=incremental (EnableIncrementalVacuum, or
-    # `bin/rails railwatch:vacuum` for a database that predates it).
-    # Measured at 80k-190k pages/s warm, so a full run is well under a second
-    # there; the slicing is for the cold, large file, where each slice takes
-    # and releases the write lock instead of holding it throughout.
-    VACUUM_PAGES_PER_SLICE = 2_000
-    # 50k pages, ~200MB at SQLite's 4KB default. A few thousand pages a night
-    # would never keep up with a night's deletes on a busy app, and the file
-    # would go on growing with the freelist; a backlog past this one still
-    # drains over successive nightly runs.
-    VACUUM_SLICES = 25
-
-    RAW = [ Telemetry::Execution, Telemetry::Query, Telemetry::Exception, Telemetry::CacheEvent, Telemetry::Mail,
-            Telemetry::Broadcast, Telemetry::Notification, Telemetry::OutgoingRequest, Telemetry::StorageOp,
-            Telemetry::ViewRender, Telemetry::Log, Telemetry::EnqueuedJob, Telemetry::Transaction,
-            Telemetry::NPlusOne, Telemetry::Deprecation, Telemetry::Visit, Telemetry::Span,
-            Telemetry::LlmCall,
-            Telemetry::Profile, Telemetry::Attachment ].freeze
-
-    # checkpoint: the WAL checkpoint mode run at the end. TRUNCATE (the
-    # default, for a dedicated worker) hands the space back to the filesystem
-    # but blocks every reader and writer while it does; PASSIVE checkpoints
-    # what it can without waiting on anyone, which is what a prune running
-    # inside a Puma worker (Railwatch::Maintenance) must use.
-    def perform(environment = nil, checkpoint: "TRUNCATE")
-      return [ Environment.current ].each { |env| self.class.perform_later(env) } if environment.nil?
-
-      cutoff = environment.retention_days.days.ago
-      aggregate_cutoff = AGGREGATE_RETENTION.ago
-      environment.with_telemetry do
-        unindex_logs(cutoff) if Telemetry::Log.fts_available?
-        RAW.each { |klass| prune(klass, cutoff) }
-        # NOT EXISTS rather than NOT IN: queries.group_hash is nullable, and one
-        # NULL in a NOT IN subquery makes it match nothing. A few hundred shapes.
-        Telemetry::QueryShape.where("NOT EXISTS (SELECT 1 FROM queries WHERE queries.group_hash = query_shapes.group_hash)").delete_all
-        prune_sessions(cutoff)
-        Telemetry::Rollup.where(bucket: ...aggregate_cutoff).delete_all
-        Telemetry::ReleaseHealth.where(bucket: ...aggregate_cutoff).in_batches(of: 5_000).delete_all
-        Telemetry::Person.where(last_seen_at: ...cutoff).in_batches(of: 5_000).delete_all
-        Telemetry::IngestBatch.where(received_at: ...cutoff).delete_all
-        Telemetry::Process.where(booted_at: ...cutoff).delete_all
-        Telemetry::HealthSample.where(sampled_at: ...cutoff).delete_all
-        # Before the checkpoint, not after: incremental_vacuum truncates the
-        # database file, and in WAL mode that truncation only reaches the file
-        # on disk once it is checkpointed.
-        TelemetryRecord.reclaim_freelist!(slice: VACUUM_PAGES_PER_SLICE, slices: VACUUM_SLICES)
-        TelemetryRecord.connection.execute("PRAGMA wal_checkpoint(#{checkpoint})") if TelemetryRecord.sqlite?
-      end
-    end
-
-    private
-
-    # in_batches walks a table by primary key: its probe is "WHERE occurred_at
-    # < ? ORDER BY id LIMIT n", which no index serves, so SQLite scanned the
-    # whole table by rowid to find nothing to delete -- 42 seconds on the
-    # platform's own 12M-row queries table, every night, for an empty result.
-    # Ordered by occurred_at the same probe is one seek on any index led by
-    # that column, and the first page of expired rows is exactly the oldest
-    # ones. Tables without such an index still scan, but they are the small
-    # ones.
-    def prune(klass, cutoff)
-      MAX_BATCHES_PER_TABLE.times do
-        # Ordered by (occurred_at, id), not occurred_at alone: rows that tie
-        # on the timestamp at the limit boundary would otherwise be a
-        # different set here than in unindex_logs above, which leaves stale
-        # FTS postings behind and withdraws postings for logs that are staying.
-        ids = klass.where(occurred_at: ...cutoff).order(:occurred_at, :id).limit(BATCH).pluck(:id)
-        break if ids.empty?
-        klass.where(id: ids).delete_all
-        break if ids.size < BATCH
-      end
-    end
-
-    # Sessions are the raw input to the hourly release_health aggregate, so they
-    # are deleted a whole hour at a time. Flooring the cutoff to the start of its
-    # hour leaves the partial hour straddling the boundary in place: deleting
-    # only its expired prefix would let the next rollup rebuild that hour from
-    # the surviving tail and silently shrink an already correct aggregate.
-    def prune_sessions(cutoff)
-      expired = Telemetry::Session.where(occurred_at: ...cutoff.utc.beginning_of_hour)
-      SESSION_HOURS_PER_RUN.times do
-        oldest = expired.minimum(:occurred_at)
-        break if oldest.nil?
-
-        hour = oldest.utc.beginning_of_hour
-        expired.where(occurred_at: hour...(hour + 1.hour)).in_batches(of: 5_000).delete_all
-      end
-    end
-
-    # logs_fts is an external-content index with no triggers, so the postings
-    # for a log line have to be withdrawn while its row (and message) is still
-    # there. Deleting the rows first would leave the index permanently out of
-    # sync -- searches would keep returning rowids that no longer exist.
-    # Bounded the same way as the rows it precedes: the postings for at most
-    # MAX_BATCHES_PER_TABLE * BATCH expired lines are withdrawn per run, which
-    # is exactly the set prune(Telemetry::Log) will delete this run.
-    def unindex_logs(cutoff)
-      Telemetry::Log.connection.execute(Telemetry::Log.sanitize_sql_array([
-        "INSERT INTO logs_fts(logs_fts, rowid, message) SELECT 'delete', id, message FROM logs " \
-        "WHERE occurred_at < ? ORDER BY occurred_at, id LIMIT ?", cutoff, MAX_BATCHES_PER_TABLE * BATCH
-      ]))
-    end
-  end
-end

data/app/jobs/railwatch/release_health_rollup_job.rb

@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-module Railwatch
-  # Recomputes one hourly release_health bucket from raw `sessions` rows.
-  # Idempotent, and debounced by the same Solid Queue concurrency control
-  # RollupJob uses: while one recompute of a bucket is queued or running,
-  # further enqueues for it are discarded rather than piling up.
-  #
-  # The gem re-sends a live session every flush interval, so the same
-  # session_id lands in a bucket many times. Each id collapses to the worst
-  # status it reached (crashed beats errored beats ok beats started) and its
-  # longest reported duration, which is what makes "sessions" a session count
-  # rather than a record count.
-  class ReleaseHealthRollupJob < ApplicationJob
-    queue_as :rollups
-    limits_concurrency to: 1, key: ->(environment, bucket) { "release_health:#{environment.id}:#{bucket.to_i}" }, duration: 10.minutes, on_conflict: :discard
-
-    RANKS = { "started" => 0, "ok" => 1, "errored" => 2, "crashed" => 3 }.freeze
-    CRASHED = RANKS["crashed"]
-    ERRORED = RANKS["errored"]
-
-    def perform(environment, bucket)
-      bucket = bucket.utc.beginning_of_hour
-      # PruneTelemetryJob deletes raw sessions in whole hours below this floor.
-      # A rollup enqueued before the prune can run after it, and rebuilding a
-      # pruned hour would replace a complete aggregate with an empty one.
-      return if bucket < environment.retention_days.days.ago.utc.beginning_of_hour
-
-      environment.with_telemetry do
-        rows = Telemetry::Session.where(occurred_at: bucket...(bucket + 1.hour))
-          .where.not(deploy: nil).pluck(:deploy, :session_id, :status, :duration, :user_ref)
-        aggregates = rows.group_by(&:first).map { |deploy, group| aggregate(deploy, bucket, collapse(group)) }
-        Telemetry::ReleaseHealth.transaction do
-          Telemetry::ReleaseHealth.where(bucket: bucket).delete_all
-          Telemetry::ReleaseHealth.insert_all(aggregates) if aggregates.any?
-        end
-      end
-    end
-
-    private
-
-    # session_id => the worst status, longest duration, and first user seen for
-    # it in this bucket.
-    def collapse(rows)
-      rows.each_with_object({}) do |(_deploy, session_id, status, duration, user_ref), sessions|
-        session = sessions[session_id] ||= { rank: 0, duration: nil, user: nil }
-        session[:rank] = [ session[:rank], RANKS.fetch(status.to_s, 0) ].max
-        session[:duration] = [ session[:duration] || 0, duration ].max if duration
-        session[:user] ||= user_ref
-      end
-    end
-
-    def aggregate(deploy, bucket, sessions)
-      durations = sessions.values.filter_map { |s| s[:duration] }
-      users = sessions.values.filter_map { |s| s[:user] }.uniq
-      crashed_users = sessions.values.select { |s| s[:rank] == CRASHED }.filter_map { |s| s[:user] }.uniq
-      { deploy: deploy, bucket: bucket, sessions: sessions.size,
-       sessions_errored: sessions.values.count { |s| s[:rank] == ERRORED },
-       sessions_crashed: sessions.values.count { |s| s[:rank] == CRASHED },
-       users: users.size, users_crashed: crashed_users.size,
-       duration_sum: durations.sum, duration_count: durations.size }
-    end
-  end
-end

data/app/jobs/railwatch/rollup_catchup_job.rb

@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Railwatch
-  # Recomputes the current and previous hour so rollups are never more than a
-  # schedule tick stale, whatever happened to the per-batch RollupJob enqueues
-  # (debounced in the web process, and their concurrency semaphore can outlive
-  # a worker restart). The platform iterates every active environment; an
-  # embedded install has one.
-  class RollupCatchupJob < ApplicationJob
-    queue_as :rollups
-
-    def perform
-      env = Environment.current
-      now = Time.current
-      [ now.beginning_of_hour, (now - 1.hour).beginning_of_hour ].each do |bucket|
-        RollupJob.perform_now(env, bucket)
-        ReleaseHealthRollupJob.perform_now(env, bucket)
-      end
-    end
-  end
-end

data/app/jobs/railwatch/rollup_job.rb

@@ -1,132 +0,0 @@
-# frozen_string_literal: true
-
-module Railwatch
-  # Recomputes the hourly rollups for one bucket from raw rows. Idempotent:
-  # runs again whenever new rows land in that hour. Debounced twice: Ingest::
-  # Batch enqueues at most one run per bucket per minute, and Solid Queue's
-  # concurrency control discards an enqueue while a run for that bucket is
-  # already queued or running (the default :block would let a busy hour pile
-  # up thousands of identical jobs). RollupCatchupJob sweeps the current and
-  # previous hour on its recurring schedule for anything either debounce
-  # dropped; an embedded install instead runs this once an hour for the
-  # previous hour from Railwatch::Maintenance, since its batches already
-  # fold themselves into the current hour as they land.
-  class RollupJob < ApplicationJob
-    queue_as :rollups
-    limits_concurrency to: 1, key: ->(environment, bucket) { "#{environment.id}:#{bucket.to_i}" }, duration: 10.minutes, on_conflict: :discard
-
-    SOURCES = {
-      "request" => [ Telemetry::Execution, ->(r) { r.requests }, ->(row) { row.name } ],
-      "job_attempt" => [ Telemetry::Execution, ->(r) { r.jobs }, ->(row) { row.name } ],
-      "scheduled_task" => [ Telemetry::Execution, ->(r) { r.scheduled }, ->(row) { row.name } ],
-      "command" => [ Telemetry::Execution, ->(r) { r.commands }, ->(row) { row.name } ],
-      "channel_action" => [ Telemetry::Execution, ->(r) { r.channels }, ->(row) { row.name } ],
-      "query" => [ Telemetry::Query, ->(r) { r }, ->(row) { row.sql.first(255) } ],
-      "outgoing_request" => [ Telemetry::OutgoingRequest, ->(r) { r }, ->(row) { "#{row.method} #{row.host}" } ],
-      "cache_event" => [ Telemetry::CacheEvent, ->(r) { r }, ->(row) { "#{row.store} #{row.key}" } ],
-      "mail" => [ Telemetry::Mail, ->(r) { r }, ->(row) { row.mailer } ],
-      "visit" => [ Telemetry::Visit, ->(r) { r }, ->(row) { row.component } ],
-      "span" => [ Telemetry::Span, ->(r) { r }, ->(row) { row.name } ],
-      "notification" => [ Telemetry::Notification, ->(r) { r }, ->(row) { row.notifier || row.delivery_method } ],
-      "view_render" => [ Telemetry::ViewRender, ->(r) { r }, ->(row) { row.identifier } ],
-      "transaction" => [ Telemetry::Transaction, ->(r) { r }, ->(row) { "#{row.connection} · #{row.outcome}" } ],
-      "llm_call" => [ Telemetry::LlmCall, ->(r) { r.models }, ->(row) { "#{row.model} · #{row.operation}" } ],
-      "llm_tool" => [ Telemetry::LlmCall, ->(r) { r.tools }, ->(row) { row.tool_name.to_s } ]
-    }.freeze
-
-    # SQLite binds at most 32,766 variables per statement; a rollup row is 14.
-    INSERT_SLICE = 500
-
-    def perform(environment, bucket)
-      bucket = bucket.utc.beginning_of_hour
-      range = bucket...(bucket + 1.hour)
-      # Recomputing a busy hour is thousands of queries; recording each of
-      # them into Railwatch's own telemetry was 70% of everything this platform
-      # ingested about itself. The job record still ships with its duration.
-      environment.with_telemetry do
-        Railwatch.ignore { recompute(environment, bucket, range) }
-      end
-    end
-
-    private
-
-    # Every group of one type is written in one transaction: the hour's
-    # rollups for that type replace each other atomically, and the write lock
-    # is held only for the writes. Rails opens each SQLite transaction
-    # IMMEDIATE, so reading the hour inside it would hold every other writer
-    # off the tenant file for the whole read -- a busy hour is tens of
-    # seconds, and ingest gets five before it gives up. On 2026-09-05 one
-    # 37-second recompute failed every batch that arrived while it ran.
-    def recompute(environment, bucket, range)
-      SOURCES.each do |type, (klass, scope, namer)|
-        Railwatch.span("rollup.#{type}", bucket: bucket.iso8601) do
-          rows = scope.call(klass).where(occurred_at: range).select(:id, :group_hash, :duration, *(klass == Telemetry::Execution ? %i[name status outcome kind] : %i[]), *(klass == Telemetry::Query ? [ Telemetry::Query::SQL ] : []), *(klass == Telemetry::OutgoingRequest ? %i[method host status_code] : []), *(klass == Telemetry::CacheEvent ? %i[store key type] : []), *(klass == Telemetry::Mail ? %i[mailer failed] : []), *(klass == Telemetry::Visit ? %i[component status] : []), *(klass == Telemetry::Notification ? %i[notifier delivery_method failed] : []), *(klass == Telemetry::ViewRender ? %i[identifier kind] : []), *(klass == Telemetry::Transaction ? %i[outcome connection] : []), *(klass == Telemetry::Span ? %i[name status] : []), *(klass == Telemetry::LlmCall ? %i[operation model tool_name status input_tokens output_tokens cache_read_tokens cache_write_tokens cost_nanos finish_reason attachments] : []))
-          groups = rows.group_by(&:group_hash).except(nil)
-          next if groups.empty?
-          rows = groups.map do |group_hash, group|
-            Telemetry::Rollup.fresh_attributes(
-              record_type: type, group_hash: group_hash, name: namer.call(group.first).to_s, bucket: bucket,
-              durations: group.map(&:duration),
-              errors: group.count { |r| error?(type, r) },
-              client_errors: group.count { |r| type == "request" && r.status.to_i.between?(400, 499) },
-              extra: extra_for(type, group))
-          end
-          # One DELETE and a few INSERTs per type instead of a SELECT, a DELETE
-          # and an INSERT per group: a busy hour of queries is 2,000 groups,
-          # and this job was 20,000 of the platform's own N+1 records a week.
-          Telemetry::Rollup.transaction do
-            # The hour was read outside this transaction (above), so a batch
-            # that landed in between has already been folded into the row
-            # about to be replaced (Ingest::RollupAbsorber). A stored count
-            # higher than the recomputed one means exactly that: keep the
-            # stored row rather than overwrite it with a snapshot that
-            # predates the batch. The next run picks the group up.
-            stored = Telemetry::Rollup.where(record_type: type, group_hash: groups.keys, bucket: bucket).pluck(:group_hash, :count).to_h
-            rows = rows.reject { |row| stored[row[:group_hash]].to_i > row[:count] }
-            next if rows.empty?
-
-            Telemetry::Rollup.where(record_type: type, group_hash: rows.map { |row| row[:group_hash] }, bucket: bucket).delete_all
-            rows.each_slice(INSERT_SLICE) { |slice| Telemetry::Rollup.insert_all(slice) }
-          end
-        end
-      end
-    end
-
-    def error?(type, row)
-      case type
-      when "request" then row.status.to_i >= 500
-      when "job_attempt", "scheduled_task", "channel_action" then row.outcome == "failed"
-      when "command" then row.status.to_i != 0
-      when "outgoing_request" then row.status_code.to_i >= 500 || row.status_code.to_i.zero?
-      when "mail" then row.failed
-      when "visit" then row.status == "error"
-      when "notification" then row.failed
-      when "transaction" then row.outcome == "rollback"
-      when "span", "llm_call", "llm_tool" then row.status == "failed"
-      else false
-      end
-    end
-
-    def extra_for(type, group)
-      case type
-      when "cache_event"
-        { hits: group.count { |r| r.type == "hit" }, misses: group.count { |r| r.type == "miss" || r.type == "generate" } }
-      when "view_render"
-        { kind: group.group_by(&:kind).max_by { |_kind, rows| rows.size }&.first }
-      when "llm_call"
-        { input_tokens: group.sum { |r| r.input_tokens.to_i }, output_tokens: group.sum { |r| r.output_tokens.to_i },
-         cache_read_tokens: group.sum { |r| r.cache_read_tokens.to_i }, cache_write_tokens: group.sum { |r| r.cache_write_tokens.to_i },
-         cost_nanos: group.sum { |r| r.cost_nanos.to_i },
-         priced: group.count { |r| r.cost_nanos },
-         # A call that failed before the provider answered used no tokens,
-         # so there is nothing to price; only an answered call can be unpriced.
-         unpriced: group.count { |r| r.cost_nanos.nil? && r.status != "failed" },
-         # A cut-off answer is not an error and will never show in the error
-         # rate, so it needs counting on its own or it stays invisible.
-         truncated: group.count { |r| r.finish_reason == "max_tokens" },
-         with_attachments: group.count { |r| r.attachments.to_i.positive? } }
-      else {}
-      end
-    end
-  end
-end

data/app/jobs/railwatch/scheduled_task_scan_job.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-module Railwatch
-  class ScheduledTaskScanJob < ApplicationJob
-    queue_as :default
-
-    def perform
-      [ Environment.current ].each { |env| CheckScheduledTasksJob.perform_later(env) }
-    end
-  end
-end

data/app/models/concerns/railwatch/detection_snapshotting.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-module Railwatch
-  # Issue.record_occurrence! replaces `sample` wholesale on every breach, but the
-  # typed detector snapshot is durable context, not the latest reading. Capturing
-  # it costs a telemetry query per detector run, so re-capture only when the
-  # issue is newly open and otherwise carry the stored snapshot forward.
-  module DetectionSnapshotting
-    private
-
-    def carry_detection(issue, outcome)
-      detection = if outcome == :new || outcome == :regressed
-        yield
-      else
-        issue.sample_previously_was&.dig("detection")
-      end
-      issue.update!(sample: issue.sample.merge("detection" => detection)) if detection
-    end
-  end
-end