railwatch 0.1.4 → 0.2.0

data/app/models/railwatch/ingest/rollup_absorber.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Ingest
+    # Folds one batch's rows into the hourly rollups as they land, so a count
+    # or a percentile on the dashboard moves with every batch instead of
+    # waiting for RollupJob to recompute the hour from raw rows. Same
+    # classification as RollupJob (which stays as the reconciler for anything
+    # this missed), applied to the mapped row hashes before they are inserted
+    # rather than to Active Record rows read back afterwards.
+    #
+    # One SELECT for the groups this batch touches, the merge in Ruby, one
+    # upsert: the read and the write both stay small whatever the hour holds,
+    # so this never grows into the 100 ms recompute that RollupJob is on a
+    # busy hour. It runs inside the batch's transaction.
+    class RollupAbsorber
+      # record_type => [table class, row filter, group name]. Executions carry
+      # their kind on the row; LLM calls split into models and tools.
+      SOURCES = {
+        "request" => [ Telemetry::Execution, ->(r) { r[:kind] == "request" }, ->(r) { r[:name] } ],
+        "job_attempt" => [ Telemetry::Execution, ->(r) { r[:kind] == "job_attempt" }, ->(r) { r[:name] } ],
+        "scheduled_task" => [ Telemetry::Execution, ->(r) { r[:kind] == "scheduled_task" }, ->(r) { r[:name] } ],
+        "command" => [ Telemetry::Execution, ->(r) { r[:kind] == "command" }, ->(r) { r[:name] } ],
+        "channel_action" => [ Telemetry::Execution, ->(r) { r[:kind] == "channel_action" }, ->(r) { r[:name] } ],
+        "query" => [ Telemetry::Query, ->(_r) { true }, ->(r) { r[:sql].to_s } ],
+        "outgoing_request" => [ Telemetry::OutgoingRequest, ->(_r) { true }, ->(r) { "#{r[:method]} #{r[:host]}" } ],
+        "cache_event" => [ Telemetry::CacheEvent, ->(_r) { true }, ->(r) { "#{r[:store]} #{r[:key]}" } ],
+        "mail" => [ Telemetry::Mail, ->(_r) { true }, ->(r) { r[:mailer] } ],
+        "visit" => [ Telemetry::Visit, ->(_r) { true }, ->(r) { r[:component] } ],
+        "span" => [ Telemetry::Span, ->(_r) { true }, ->(r) { r[:name] } ],
+        "notification" => [ Telemetry::Notification, ->(_r) { true }, ->(r) { r[:notifier] || r[:delivery_method] } ],
+        "view_render" => [ Telemetry::ViewRender, ->(_r) { true }, ->(r) { r[:identifier] } ],
+        "transaction" => [ Telemetry::Transaction, ->(_r) { true }, ->(r) { "#{r[:connection]} · #{r[:outcome]}" } ],
+        "llm_call" => [ Telemetry::LlmCall, ->(r) { r[:operation] != Telemetry::LlmCall::TOOL }, ->(r) { "#{r[:model]} · #{r[:operation]}" } ],
+        "llm_tool" => [ Telemetry::LlmCall, ->(r) { r[:operation] == Telemetry::LlmCall::TOOL }, ->(r) { r[:tool_name].to_s } ]
+      }.freeze
+
+      # SQLite binds at most 32,766 variables per statement; a rollup row is 14.
+      UPSERT_SLICE = 500
+
+      def initialize(rows_by_class, query_shapes: {})
+        @rows_by_class = rows_by_class
+        # Writer files a query's text on its shape and blanks the row, so the
+        # group's name has to come from the shape when the row is empty.
+        @query_shapes = query_shapes
+      end
+
+      # Groups every rolled-up row by (type, group, hour), merges each group
+      # into its existing rollup row if there is one, and writes them back.
+      def absorb!
+        pending = collect
+        return 0 if pending.empty?
+
+        existing = load_existing(pending.keys)
+        rows = pending.map { |key, group| merged_row(key, group, existing[key]) }
+        rows.each_slice(UPSERT_SLICE) do |slice|
+          Telemetry::Rollup.upsert_all(slice, unique_by: %i[record_type group_hash bucket], record_timestamps: false)
+        end
+        rows.size
+      end
+
+      private
+
+      # { [type, group_hash, bucket] => { name:, durations:, errors:, client_errors:, extra: } }
+      def collect
+        pending = {}
+        SOURCES.each do |type, (klass, filter, namer)|
+          @rows_by_class.fetch(klass, []).each do |row|
+            next unless filter.call(row)
+            group_hash = row[:group_hash]
+            duration = row[:duration]
+            next if group_hash.nil? || duration.nil?
+
+            bucket = bucket_for(row[:occurred_at])
+            next unless bucket
+
+            entry = pending[[ type, group_hash, bucket ]] ||= { name: nil, durations: [], errors: 0, client_errors: 0, extra: Hash.new(0) }
+            entry[:name] ||= name_for(type, row, namer)
+            entry[:durations] << duration.to_i
+            entry[:errors] += 1 if error?(type, row)
+            entry[:client_errors] += 1 if type == "request" && row[:status].to_i.between?(400, 499)
+            add_extra(type, row, entry[:extra])
+          end
+        end
+        pending
+      end
+
+      def load_existing(keys)
+        found = {}
+        keys.group_by { |type, _group, bucket| [ type, bucket ] }.each do |(type, bucket), group_keys|
+          Telemetry::Rollup.where(record_type: type, bucket: bucket, group_hash: group_keys.map { |k| k[1] })
+                           .each { |row| found[[ type, row.group_hash, bucket ]] = row }
+        end
+        found
+      end
+
+      def merged_row((type, group_hash, bucket), group, existing)
+        digest = existing&.digest ? TDigest::TDigest.from_bytes(existing.digest) : TDigest::TDigest.new(0.01)
+        group[:durations].each { |d| digest.push(d) }
+        digest.compress!
+        extra = existing ? existing.extra.merge(group[:extra]) { |_k, a, b| a.is_a?(Numeric) && b.is_a?(Numeric) ? a + b : b } : group[:extra].to_h
+        {
+          record_type: type, group_hash: group_hash, bucket: bucket,
+          name: (group[:name].presence || existing&.name || "").to_s.first(255),
+          count: existing&.count.to_i + group[:durations].size,
+          error_count: existing&.error_count.to_i + group[:errors],
+          client_error_count: existing&.client_error_count.to_i + group[:client_errors],
+          duration_sum: existing&.duration_sum.to_i + group[:durations].sum,
+          duration_max: [ existing&.duration_max.to_i, group[:durations].max ].max,
+          p50: digest.percentile(0.5).to_i, p95: digest.percentile(0.95).to_i, p99: digest.percentile(0.99).to_i,
+          digest: digest.as_small_bytes, extra: extra
+        }
+      end
+
+      # Mapper writes occurred_at as a UTC string; the hour is its prefix.
+      def bucket_for(occurred_at)
+        return nil if occurred_at.nil?
+
+        @buckets ||= {}
+        @buckets[occurred_at.to_s[0, 13]] ||= Time.zone.parse(occurred_at.to_s)&.utc&.beginning_of_hour
+      end
+
+      def name_for(type, row, namer)
+        if type == "query"
+          text = row[:sql].to_s
+          text = @query_shapes[row[:group_hash]].to_s if text.empty?
+          text.first(255)
+        else
+          namer.call(row).to_s
+        end
+      end
+
+      def error?(type, row)
+        case type
+        when "request" then row[:status].to_i >= 500
+        when "job_attempt", "scheduled_task", "channel_action" then row[:outcome] == "failed"
+        when "command" then row[:status].to_i != 0
+        when "outgoing_request" then row[:status_code].to_i >= 500 || row[:status_code].to_i.zero?
+        when "mail", "notification" then row[:failed] ? true : false
+        when "visit" then row[:status] == "error"
+        when "transaction" then row[:outcome] == "rollback"
+        when "span", "llm_call", "llm_tool" then row[:status] == "failed"
+        else false
+        end
+      end
+
+      def add_extra(type, row, extra)
+        case type
+        when "cache_event"
+          extra["hits"] += 1 if row[:type] == "hit"
+          extra["misses"] += 1 if row[:type] == "miss" || row[:type] == "generate"
+        when "view_render"
+          # RollupJob stores the hour's commonest kind; a batch cannot know
+          # that, so the latest seen wins. Same shape, close enough.
+          extra["kind"] = row[:kind] if row[:kind]
+        when "llm_call"
+          extra["input_tokens"] += row[:input_tokens].to_i
+          extra["output_tokens"] += row[:output_tokens].to_i
+          extra["cache_read_tokens"] += row[:cache_read_tokens].to_i
+          extra["cache_write_tokens"] += row[:cache_write_tokens].to_i
+          extra["cost_nanos"] += row[:cost_nanos].to_i
+          extra["priced"] += 1 if row[:cost_nanos]
+          extra["unpriced"] += 1 if row[:cost_nanos].nil?
+          extra["truncated"] += 1 if row[:finish_reason] == "max_tokens"
+          extra["with_attachments"] += 1 if row[:attachments].to_i.positive?
+        end
+      end
+    end
+  end
+end

data/app/models/railwatch/ingest/writer.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Ingest
+    # Inserts mapped rows (klass => [row_hash, ...], from Ingest::Mapper.row_for)
+    # into the current tenant's telemetry database inside one transaction.
+    #
+    # On SQLite, binds straight into a prepared statement per (table, column
+    # set) instead of going through Active Record's insert machinery — this is
+    # the fast path the platform is tuned for. On any other adapter (Postgres
+    # stays supported per SCOPE.md), falls back to Active Record's insert_all.
+    class Writer
+      # Classes whose new rowids we need after the insert: exception ids go
+      # back to the caller for grouping, log ids into the full-text index.
+      ID_CLASSES = [ Telemetry::Exception, Telemetry::Log ].freeze
+
+      def initialize(rows_by_class)
+        @rows_by_class = rows_by_class
+      end
+
+      # group_hash => statement, for the queries this batch filed on their
+      # shape; RollupAbsorber names a query group from it.
+      attr_reader :query_shapes
+
+      def write!
+        exception_ids = []
+        TelemetryRecord.transaction do
+          tables = with_query_shapes(@rows_by_class)
+          connection = TelemetryRecord.connection
+          exception_ids = connection.adapter_name == "SQLite" ? write_sqlite(connection, tables) : write_generic(tables)
+        end
+        exception_ids
+      end
+
+      private
+
+      # A query's group is the gem's digest of its connection name and its
+      # normalized statement, which is the text the gem sends unless it is
+      # configured to capture values. A row whose text digests to its group
+      # therefore carries the group's statement: it is filed once in
+      # query_shapes and the row stores "". Any other text (captured values,
+      # a statement the gem truncated, an older gem's raw text) stays on the
+      # row. One MD5 per row; the shapes ride the same INSERT OR IGNORE as
+      # every table, so a group already on file costs nothing.
+      def with_query_shapes(rows_by_class)
+        shapes = {}
+        rows_by_class.fetch(Telemetry::Query, []).each do |row|
+          group_hash = row[:group_hash]
+          next unless group_hash && row[:sql].present? && Railwatch::Record.group_hash(row[:connection], row[:sql]) == group_hash
+
+          shapes[group_hash] ||= { group_hash: group_hash, sql: row[:sql] }
+          row[:sql] = ""
+        end
+        @query_shapes = shapes.transform_values { |shape| shape[:sql] }
+        shapes.empty? ? rows_by_class : rows_by_class.merge(Telemetry::QueryShape => shapes.values)
+      end
+
+      def write_sqlite(connection, tables)
+        raw = connection.raw_connection
+        ids = Hash.new { |h, k| h[k] = [] }
+        tables.each do |klass, rows|
+          next if rows.empty?
+          table = connection.quote_table_name(klass.table_name)
+          rows.group_by(&:keys).each do |columns, group_rows|
+            ids[klass].concat(insert_group(raw, connection, table, columns, group_rows, klass))
+          end
+        end
+        index_logs(raw, ids[Telemetry::Log])
+        ids[Telemetry::Exception]
+      end
+
+      def insert_group(raw, connection, table, columns, rows, klass)
+        column_sql = columns.map { |c| connection.quote_column_name(c.to_s) }.join(",")
+        placeholders = ([ "?" ] * columns.size).join(",")
+        # The gem re-sends a batch after a transport timeout, so the same
+        # execution can arrive twice; the unique index on execution_id turns
+        # the duplicate into a no-op instead of aborting the whole batch.
+        stmt = raw.prepare("INSERT OR IGNORE INTO #{table} (#{column_sql}) VALUES (#{placeholders})")
+        ids = []
+        begin
+          rows.each do |row|
+            stmt.bind_params(*columns.map { |c| row[c] })
+            stmt.step
+            ids << raw.last_insert_row_id if ID_CLASSES.include?(klass) && raw.changes.positive?
+            stmt.reset!
+          end
+        ensure
+          stmt.close
+        end
+        ids
+      end
+
+      # logs_fts is an external-content FTS5 table declared without triggers
+      # (the schema dumper cannot carry them), so every inserted message has to
+      # be copied into the index by hand, inside the same transaction as the
+      # rows themselves. Only SQLite has the index; write_generic does nothing.
+      def index_logs(raw, ids)
+        # A tenant database migrated before the index existed, or one restored
+        # from a backup that predates it, must not fail every batch that carries
+        # a log line; telemetry:fts:rebuild creates the index later.
+        return if ids.empty? || !Telemetry::Log.fts_available?
+
+        ids.each_slice(500) do |slice|
+          placeholders = ([ "?" ] * slice.size).join(",")
+          raw.execute("INSERT INTO logs_fts(rowid, message) SELECT id, message FROM logs WHERE id IN (#{placeholders})", slice)
+        end
+      end
+
+      def write_generic(tables)
+        exception_ids = []
+        tables.each do |klass, rows|
+          next if rows.empty?
+          result = klass.insert_all(restore_json(klass, rows), record_timestamps: false,
+                                    unique_by: (:execution_id if klass == Telemetry::Execution),
+                                    returning: klass == Telemetry::Exception ? [ :id ] : false)
+          exception_ids.concat(result.rows.flatten) if klass == Telemetry::Exception
+        end
+        exception_ids
+      end
+
+      # insert_all round-trips every value through its column's type (cast
+      # then serialize) before quoting it, which is correct for booleans
+      # (0/1 casts back to false/true) and timestamps (the formatted string
+      # parses back to a Time) but not for a JSON column: its mutable type
+      # re-encodes a String as-is instead of parsing it, which would double-
+      # encode our pre-serialized JSON. Only this fallback needs the fix —
+      # the SQLite fast path writes the encoded string straight through.
+      def restore_json(klass, rows)
+        json_columns = Ingest::Mapper.columns_for(klass).select { |_, (type, _)| type == :json }.keys.map(&:to_sym)
+        return rows if json_columns.empty?
+        rows.map do |row|
+          row.dup.tap { |r| json_columns.each { |c| r[c] = JSON.parse(r[c]) if r[c].is_a?(String) } }
+        end
+      end
+    end
+  end
+end

data/app/models/railwatch/issue.rb

@@ -0,0 +1,277 @@
+# frozen_string_literal: true
+
+module Railwatch
+  # A grouped exception or performance problem with a lifecycle. Lives in the
+  # primary database so it keeps its sequential id after telemetry is pruned.
+  class Issue < ApplicationRecord
+    self.table_name = "railwatch_issues"
+    class InvalidMerge < ArgumentError; end
+
+    KINDS = %w[exception performance anomaly].freeze
+    STATUSES = %w[open resolved ignored merged].freeze
+    PRIORITIES = %w[low normal high urgent].freeze
+    # How many recent occurrences a split by message looks at.
+    SPLIT_SAMPLE = 200
+    MERGE_CHAIN_LIMIT = 10
+
+    def application = Application.current
+    def environment = Environment.current
+    # assignee is a plain id on an embedded install (User is not a table).
+    def assignee
+      assignee_id && User.find_by(id: assignee_id)
+    end
+
+    def assignee=(user)
+      self.assignee_id = user&.id&.to_s
+    end
+    belongs_to :merged_into, class_name: "Issue", optional: true
+    has_many :merged_issues, class_name: "Issue", foreign_key: :merged_into_id, inverse_of: :merged_into, dependent: :nullify
+    has_many :comments, dependent: :destroy
+    has_many :alerts, dependent: :nullify
+    has_many :activities, class_name: "IssueActivity", dependent: :destroy
+
+    validates :kind, inclusion: { in: KINDS }
+    validates :status, inclusion: { in: STATUSES }
+    validates :priority, inclusion: { in: PRIORITIES }
+    validates :group_hash, presence: true, uniqueness: { scope: :environment_id }
+    validates :title, presence: true
+
+    before_validation :assign_number, on: :create
+    after_create :record_created_activity
+    after_update :record_priority_activity
+    after_update :record_assignee_activity
+
+    scope :open, -> { where(status: "open") }
+    scope :recent, -> { order(last_seen_at: :desc) }
+
+    def key
+      "#{application.issue_prefix}-#{number}"
+    end
+
+    def open? = status == "open"
+    def resolved? = status == "resolved"
+    def merged? = status == "merged"
+
+    def resolve!(deploy: nil)
+      record_status_change! { update!(status: "resolved", resolved_at: Time.current, resolved_in_deploy: deploy) }
+    end
+
+    def ignore!
+      record_status_change! { update!(status: "ignored") }
+    end
+
+    def reopen!
+      record_status_change! { update!(status: "open", resolved_at: nil, resolved_in_deploy: nil) }
+    end
+
+    # Merges this issue into `target`: moves comments over, folds occurrence
+    # and affected-user counts into the target, widens its first/last seen
+    # range, and marks this issue "merged". Future occurrences for this
+    # issue's group_hash are then credited to the target (see
+    # .record_occurrence!). Same environment only.
+    def merge_into!(target)
+      raise InvalidMerge, "both issues must be persisted before merging" unless persisted? && target&.persisted?
+
+      transaction do
+        # Every merge locks the same pair in the same order. Besides preventing
+        # reciprocal requests from creating a cycle, lock! reloads both rows so
+        # folding counts cannot overwrite a merge committed from a stale object.
+        [ self, target ].sort_by(&:id).each(&:lock!)
+        validate_merge_target!(target)
+
+        comments.update_all(issue_id: target.id)
+        target.update!(occurrences: target.occurrences + occurrences, affected_users: [ target.affected_users, affected_users ].max,
+          first_seen_at: [ target.first_seen_at, first_seen_at ].min, last_seen_at: [ target.last_seen_at, last_seen_at ].max)
+        update!(status: "merged", merged_into_id: target.id)
+        activities.create!(kind: "merge", user: Viewer.user, data: { target_id: target.id, target_key: target.key })
+        target.activities.create!(kind: "absorbed", user: Viewer.user, data: { source_id: id, source_key: key, occurrences: occurrences })
+      end
+    end
+
+    # Gives the merged-in events back: occurrences counted on the target since
+    # the merge stay there (they were credited under the target's own group),
+    # only the count that came along at merge time moves back. Chained merges
+    # must be unwound from the live end so an inner source cannot be restored
+    # while its count is still included farther up the chain.
+    def unmerge!
+      transaction do
+        target_id = self.class.where(id: id).pick(:merged_into_id)
+        raise InvalidMerge, "cannot unmerge an issue that is not merged" unless target_id
+
+        target = self.class.find(target_id)
+        [ self, target ].sort_by(&:id).each(&:lock!)
+        unless merged? && merged_into_id == target.id
+          raise InvalidMerge, "issue merge changed; reload and try again"
+        end
+        if target.merged? || target.merged_into_id.present?
+          raise InvalidMerge, "cannot unmerge this issue yet; unmerge #{target.key} first (top-down order)"
+        end
+
+        target.update!(occurrences: [ target.occurrences - occurrences, 0 ].max)
+        update!(status: "open", merged_into_id: nil)
+        activities.create!(kind: "unmerge", user: Viewer.user, data: { target_id: target.id, target_key: target.key })
+      end
+    end
+
+    # Splits this issue by the raw message of its recent occurrences: one new
+    # issue per distinct message, each taking that message's count, seen range
+    # and sample, with this issue giving those occurrences up. Nothing moves in
+    # the telemetry database -- the rows are still grouped under the hash the
+    # gem sent -- so this separates what has already been seen; to split future
+    # occurrences too, give the error a real fingerprint (Railwatch.fingerprint).
+    # Returns the issues it created: [] when every recent occurrence carries
+    # the same message, or when they have all been split off already.
+    def split!(by: "message")
+      return [] unless by == "message"
+
+      rows = environment.with_telemetry { Telemetry::Exception.where(group_hash: group_hash).recent.limit(SPLIT_SAMPLE).to_a }
+      groups = rows.group_by { |row| row.message.to_s }
+      return [] if groups.size < 2
+
+      transaction do
+        pending = groups.reject { |message, _| environment.issues.exists?(group_hash: self.class.split_group_hash(group_hash, message)) }
+        created = pending.map { |message, message_rows| split_off(message, message_rows) }
+        next [] if created.empty?
+
+        moved = created.sum(&:occurrences)
+        update!(occurrences: [ occurrences - moved, 0 ].max)
+        activities.create!(kind: "split", user: Viewer.user, data: { by: by, occurrences: moved, into: created.map(&:key) })
+        created
+      end
+    end
+
+    # Mirrors the gem's Record.group_hash (MD5, 32 hex chars) but joins on a
+    # separator the gem never sends, so an issue split here can never collide
+    # with one the gem itself would group.
+    def self.split_group_hash(group_hash, message)
+      Digest::MD5.hexdigest([ group_hash, message ].join("\x1f"))[0, 32]
+    end
+
+    # Called by the grouper for every new occurrence. Returns :new, :regressed,
+    # or :seen so the caller can decide whether to alert.
+    def self.record_occurrence!(environment:, group_hash:, kind:, title:, culprit:, occurred_at:, deploy:, user_ref:, sample:, source: nil)
+      issue = find_or_initialize_by(environment_id: environment.id, group_hash: group_hash)
+      return record_occurrence_on!(issue, environment:, kind:, title:, culprit:, occurred_at:, deploy:, sample:, source:) if issue.new_record?
+
+      visited = {}
+      0.upto(MERGE_CHAIN_LIMIT) do |depth|
+        raise InvalidMerge, "cycle detected in issue merge chain" if visited[issue.id]
+        visited[issue.id] = true
+
+        next_id = nil
+        result = nil
+        # Lock only the row currently being inspected. If it has merged, release
+        # that lock before following the pointer; this avoids taking chain locks
+        # in an order that can deadlock with merge_into!.
+        issue.with_lock do
+          if issue.merged? || issue.merged_into_id.present?
+            next_id = issue.merged_into_id
+            raise InvalidMerge, "merged issue has no merge target" unless next_id
+          else
+            result = record_occurrence_on!(issue, environment:, kind:, title:, culprit:, occurred_at:, deploy:, sample:, source:)
+          end
+        end
+        return result if result
+        raise InvalidMerge, "issue merge chain exceeds #{MERGE_CHAIN_LIMIT} links" if depth == MERGE_CHAIN_LIMIT
+
+        issue = find_by(id: next_id)
+        raise InvalidMerge, "issue merge target no longer exists" unless issue
+      end
+    end
+
+    # Routes an issue lifecycle event through every alert rule subscribed to it
+    # whose filters match. Public so detectors (DetectAnomaliesJob) can route
+    # their own events through the same path.
+    def fire_alerts!(event, extra = {})
+      return unless event
+      application.alert_rules.where(event: event).find_each do |rule|
+        payload = { issue_key: key, title: title, environment: environment.name, actor: Viewer.user&.name }.merge(extra)
+        next unless rule.matches?(issue: self, payload: payload)
+        rule.fire!(event: event, issue: self, payload: payload)
+      end
+    end
+
+    # During the rollout the legacy one-workspace columns may already contain
+    # a link. Materialize it once so old rows enter the per-workspace model
+    # without losing synchronization.
+    def linear_links_with_legacy = []
+
+    private
+
+    def self.record_occurrence_on!(issue, environment:, kind:, title:, culprit:, occurred_at:, deploy:, sample:, source:)
+      outcome =
+        if issue.new_record?
+          issue.assign_attributes(application_id: environment.application.id, kind: kind, title: title, culprit: culprit,
+                                  first_seen_at: occurred_at)
+          :new
+        elsif issue.resolved? && (issue.resolved_in_deploy.nil? || deploy.to_s != issue.resolved_in_deploy)
+          issue.assign_attributes(status: "open", regressed_at: occurred_at, resolved_at: nil, resolved_in_deploy: nil)
+          :regressed
+        else
+          :seen
+        end
+      issue.occurrences += 1
+      issue.last_seen_at = occurred_at if issue.last_seen_at.nil? || occurred_at > issue.last_seen_at
+      issue.sample = sample
+      # Where the latest occurrence was raised: "browser" for a JavaScript
+      # error the beacon reported, the Rails source for everything else.
+      issue.source = source
+      issue.title = title if outcome == :new
+      issue.save!
+      issue.activities.create!(kind: "regressed", data: { at: occurred_at }) if outcome == :regressed
+      [ issue, outcome ]
+    end
+    private_class_method :record_occurrence_on!
+
+    def validate_merge_target!(target)
+      raise InvalidMerge, "can only merge issues in the same environment" unless target.environment_id == environment_id
+      raise InvalidMerge, "cannot merge an issue into itself" if target.id == id
+      raise InvalidMerge, "cannot merge an issue that is already merged" if merged? || merged_into_id.present?
+      raise InvalidMerge, "merge target must be open" unless target.open?
+      raise InvalidMerge, "merge target must not itself be merged" if target.merged_into_id.present?
+    end
+
+    def split_off(message, rows)
+      latest = rows.max_by(&:occurred_at)
+      issue = environment.issues.create!(application_id: application.id, kind: kind, culprit: culprit, source: source,
+        group_hash: self.class.split_group_hash(group_hash, message),
+        title: "#{latest.class_name}: #{message.first(200)}", occurrences: rows.size,
+        first_seen_at: rows.min_by(&:occurred_at).occurred_at, last_seen_at: latest.occurred_at,
+        sample: { exception_id: latest.id, handled: latest.handled, execution_id: latest.execution_id,
+                  execution_preview: latest.execution_preview, deploy: latest.deploy,
+                  fingerprint: latest.fingerprint, fingerprint_source: latest.fingerprint_source })
+      issue.activities.create!(kind: "split", user: Viewer.user, data: { from_id: id, from_key: key, occurrences: rows.size })
+      issue
+    end
+
+    def record_status_change!
+      from = status
+      yield
+      return if status == from
+      activities.create!(kind: "status", user: Viewer.user, data: { from: from, to: status })
+      fire_alerts!({ "resolved" => "resolved_issue", "ignored" => "ignored_issue", "open" => "regressed_issue" }[status])
+    end
+
+    def record_created_activity
+      activities.create!(kind: "created", user: Viewer.user)
+    end
+
+    def record_priority_activity
+      return unless saved_change_to_priority?
+      from, to = saved_change_to_priority
+      activities.create!(kind: "priority", user: Viewer.user, data: { from: from, to: to })
+    end
+
+    def record_assignee_activity
+      return unless saved_change_to_assignee_id?
+      from, to = saved_change_to_assignee_id
+      activities.create!(kind: "assignee", user: Viewer.user, data: { from_id: from, to_id: to })
+      fire_alerts!("assigned_issue", assignee: assignee&.name, assignee_id: to) if to
+    end
+
+
+    def assign_number
+      self.number ||= application.next_issue_number!
+    end
+  end
+end

data/app/models/railwatch/issue_activity.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Railwatch
+  # One entry in an issue's timeline: a comment or a system event (status
+  # change, priority change, reassignment, merge/unmerge, split, regression,
+  # alert sent). `data` holds kind-specific details the frontend renders into a
+  # sentence.
+  class IssueActivity < ApplicationRecord
+    self.table_name = "railwatch_issue_activities"
+    KINDS = %w[created status priority assignee comment merge absorbed unmerge split regressed alert agent].freeze
+
+    belongs_to :issue
+    def user
+      viewer_id && User.find_by(id: viewer_id)
+    end
+
+    def user=(u)
+      self.viewer_id = u&.id&.to_s
+    end
+
+    validates :kind, inclusion: { in: KINDS }
+  end
+end