railwatch 0.1.4 → 0.2.0

data/app/models/railwatch/telemetry/n_plus_one.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Telemetry
+    class NPlusOne < TelemetryRecord
+      include Child
+
+      # A bound value in the SQL the gem stores. Railwatch::SqlNormalizer has
+      # already rewritten literals to "?" and one-element IN lists to "IN (?)"
+      # before an n+1 record is written; the bare-number branch only covers SQL
+      # that reached us unnormalized.
+      BIND = /(?:=\s*(?:\?|-?\d+)|IN\s*\(\s*(?:\?|-?\d+)\s*\))/i
+
+      # Matched against `shape`, which strips identifier quoting so one set of
+      # patterns covers SQLite/Postgres ("comments") and MySQL (`comments`).
+      # POLYMORPHIC has to be tried before CHILD_LOAD: an imageable_id filter
+      # matches both, and only the first one reads it as a polymorphic owner.
+      POLYMORPHIC = /\ASELECT\s+(?:\w+\.)?\*\s+FROM\s+(\w+)\s+WHERE\s+(?:\w+\.)?(\w+)_id\s*#{BIND}\s+AND\s+(?:\w+\.)?\2_type\s*#{BIND}/i
+      COUNT_LOAD = /\ASELECT\s+COUNT\(\*\)\s+(?:AS\s+\w+\s+)?FROM\s+(\w+)\s+WHERE\s+(?:\w+\.)?(\w+)_id\s*#{BIND}/i
+      EXISTS_LOAD = /\ASELECT\s+(?:1\s+AS\s+one|EXISTS\s*\(\s*SELECT\s+1)\b.*?\sFROM\s+(\w+)\s+WHERE\s+(?:\w+\.)?(\w+)_id\s*#{BIND}/i
+      CHILD_LOAD = /\ASELECT\s+(?:\w+\.)?\*\s+FROM\s+(\w+)\s+WHERE\s+(?:\w+\.)?(\w+)_id\s*#{BIND}/i
+      PARENT_LOAD = /\ASELECT\s+(?:\w+\.)?\*\s+FROM\s+(\w+)\s+WHERE\s+(?:\w+\.)?id\s*#{BIND}/i
+
+      def timeline_label
+        sql.to_s.first(120)
+      end
+
+      # A concrete fix for the repeated statement, read off its SQL shape:
+      # {kind:, parent:, association:, code:, explanation:}, or nil when the
+      # statement isn't a shape a preload or counter cache can fix.
+      def suggestion
+        s = shape
+        polymorphic_load(s) || count_load(s) || exists_load(s) || child_load(s) || parent_load(s)
+      end
+
+      private
+
+      def shape
+        sql.to_s.delete('"`').squeeze(" ").strip
+      end
+
+      def polymorphic_load(s)
+        m = POLYMORPHIC.match(s) or return nil
+        table, as = m[1], m[2]
+        { kind: "polymorphic", parent: nil, association: table,
+          code: ".includes(:#{table})",
+          explanation: "Each owner loads its #{table} in a separate query. #{table} is polymorphic (belongs_to :#{as}), " \
+                       "so add .includes(:#{table}) on the collection you iterate#{at_source}." }
+      end
+
+      def count_load(s)
+        m = COUNT_LOAD.match(s) or return nil
+        table, parent = m[1], m[2].classify
+        { kind: "counter_cache", parent: parent, association: table,
+          code: "# app/models/#{m[1].singularize}.rb\nbelongs_to :#{m[2]}, counter_cache: true\n\n# then read\n#{m[2]}.#{table}_count",
+          explanation: "Each #{parent} counts its #{table} with its own COUNT query#{at_source}. " \
+                       "A counter cache keeps the total on #{parent.tableize}.#{table}_count, so #{m[2]}.#{table}_count needs no query." }
+      end
+
+      def exists_load(s)
+        m = EXISTS_LOAD.match(s) or return nil
+        table, parent = m[1], m[2].classify
+        { kind: "exists", parent: parent, association: table,
+          code: "#{parent}.includes(:#{table})\n\n# then\n#{m[2]}.#{table}.any?",
+          explanation: "Each #{parent} checks whether it has #{table} with its own query#{at_source}. " \
+                       "Preload with #{parent}.includes(:#{table}) and call .any? on the loaded collection, or add a counter cache." }
+      end
+
+      def child_load(s)
+        m = CHILD_LOAD.match(s) or return nil
+        table, parent = m[1], m[2].classify
+        { kind: "has_many", parent: parent, association: table,
+          code: "#{parent}.includes(:#{table})",
+          explanation: "Each #{parent} loads its #{table} in a separate query#{at_source}. " \
+                       "Preload them with #{parent}.includes(:#{table}) where the #{parent.tableize} are loaded." }
+      end
+
+      def parent_load(s)
+        m = PARENT_LOAD.match(s) or return nil
+        association = m[1].singularize
+        { kind: "belongs_to", parent: nil, association: association,
+          code: ".includes(:#{association})",
+          explanation: "Each row loads its #{association} one at a time. " \
+                       "Add .includes(:#{association}) on the collection you iterate#{at_source}." }
+      end
+
+      def at_source
+        source.present? ? " at #{source}" : ""
+      end
+    end
+  end
+end

data/app/models/railwatch/telemetry/notification.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Telemetry
+    class Notification < TelemetryRecord
+      include Child
+
+      def timeline_label
+        notifier.to_s.first(120)
+      end
+    end
+  end
+end

data/app/models/railwatch/telemetry/outgoing_request.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Telemetry
+    class OutgoingRequest < TelemetryRecord
+      include Child
+
+      def timeline_label
+        url.to_s.first(120)
+      end
+    end
+  end
+end

data/app/models/railwatch/telemetry/person.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Telemetry
+    # A monitored application's user, as seen in telemetry. Keyed by ref
+    # ("tenant:id" or "id") so the same person across tenants stays distinct.
+    class Person < TelemetryRecord
+      scope :recent, -> { order(last_seen_at: :desc) }
+
+      def self.touch_from_record(rec, at:)
+        touch_all([ [ rec, at ] ])
+        find_by(ref: rec["id"].to_s)
+      end
+
+      # One upsert per batch instead of a find-and-save per user record: the
+      # gem emits a user record once per user per hour per process, so a busy
+      # batch carries dozens. ON CONFLICT keeps first_seen_at and only
+      # overwrites name/email/tenant when the new value is present.
+      def self.touch_all(pairs)
+        rows = pairs.filter_map do |rec, at|
+          ref = rec["id"].to_s
+          next if ref.empty?
+          { ref: ref, name: rec["name"].presence, email: rec["email"].presence, app_tenant: rec["tenant"].presence,
+            first_seen_at: at, last_seen_at: at, requests_count: 0, exceptions_count: 0 }
+        end
+        return 0 if rows.empty?
+
+        # Latest timestamp wins when the same ref repeats within a batch.
+        rows = rows.group_by { |r| r[:ref] }.map { |_ref, dup| dup.max_by { |r| r[:last_seen_at] } }
+        upsert_all(rows, unique_by: :ref, record_timestamps: false,
+                   on_duplicate: Arel.sql(<<~SQL.squish))
+                     name = COALESCE(excluded.name, people.name),
+                     email = COALESCE(excluded.email, people.email),
+                     app_tenant = COALESCE(excluded.app_tenant, people.app_tenant),
+                     first_seen_at = COALESCE(people.first_seen_at, excluded.first_seen_at),
+                     last_seen_at = MAX(people.last_seen_at, excluded.last_seen_at)
+                   SQL
+        rows.size
+      end
+
+      def executions
+        Execution.where(user_ref: ref)
+      end
+
+      def display_name
+        name.presence || email.presence || ref
+      end
+    end
+  end
+end

data/app/models/railwatch/telemetry/process.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Telemetry
+    class Process < TelemetryRecord
+      self.table_name = "processes"
+      scope :recent, -> { order(booted_at: :desc) }
+    end
+  end
+end

data/app/models/railwatch/telemetry/profile.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Telemetry
+    # A sampled stack profile of one execution. `stacks` is gzip of collapsed
+    # stacks ("outer;inner;leaf count\n"), the format flamegraph tools read.
+    class Profile < TelemetryRecord
+      include Child
+
+      # Railwatch's profiler caps collapsed stacks at 4 MiB before compression.
+      # Enforce the same contract here even when a client lies about its size.
+      MAX_INGEST_BYTES = 4.megabytes
+
+      # A minute of wall profiling can decompress to tens of megabytes; the
+      # page only ever renders a flamegraph, so the payload is cut here rather
+      # than shipped whole to the browser.
+      MAX_COLLAPSED_BYTES = 2.megabytes
+
+      def collapsed
+        BoundedGzip.utf8!(BoundedGzip.decompress(stacks, max_bytes: MAX_INGEST_BYTES))
+      end
+
+      # [text, truncated?] - cut on a line boundary so the parser never sees
+      # half a stack line.
+      def collapsed_capped(limit = MAX_COLLAPSED_BYTES)
+        text, truncated = BoundedGzip.decompress_capped(stacks, limit: [ limit, MAX_INGEST_BYTES ].min)
+        text = text.sub(/[^\n]*\z/, "") if truncated
+
+        [ BoundedGzip.utf8!(text), truncated ]
+      end
+
+      def timeline_label
+        "#{profiler} profile (#{samples} samples)"
+      end
+    end
+  end
+end

data/app/models/railwatch/telemetry/query.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Telemetry
+    class Query < TelemetryRecord
+      include Child
+
+      # Raised when a row's SQL is read on a relation that did not resolve it.
+      class SqlNotResolved < StandardError; end
+
+      # The statement, from the row or from its group's shape (QueryShape).
+      # A correlated lookup rather than a join: the queries page sorts a
+      # window of millions of rows before taking 50, and a LEFT JOIN there
+      # was paid per candidate row (47 s against 0.8 s on the 5M-row window).
+      SQL = Arel.sql("COALESCE(NULLIF(queries.sql, ''), " \
+                     "(SELECT query_shapes.sql FROM query_shapes WHERE query_shapes.group_hash = queries.group_hash), '') AS resolved_sql").freeze
+
+      # Rows with their SQL resolved in the same statement, so a page loaded
+      # inside Environment#with_telemetry can be rendered after the block.
+      # Replaces the select list: count with count(:all), and pluck
+      # resolved_sql, not sql.
+      scope :with_sql, -> { select("queries.*", SQL) }
+
+      # Free-text search over the statement, wherever it is stored.
+      scope :matching, ->(text) {
+        pattern = "%#{sanitize_sql_like(text)}%"
+        where("queries.sql LIKE :pattern ESCAPE '\\' OR queries.name LIKE :pattern ESCAPE '\\' " \
+              "OR queries.group_hash IN (SELECT group_hash FROM query_shapes WHERE sql LIKE :pattern ESCAPE '\\')", pattern: pattern)
+      }
+
+      def self.timeline_scope(execution_id)
+        in_execution(execution_id).with_sql
+      end
+
+      # Always a String. A row whose text lives on its shape can only answer
+      # through with_sql; asking without it is a bug, not an empty statement.
+      def sql
+        return self[:resolved_sql].to_s if has_attribute?(:resolved_sql)
+
+        text = self[:sql].to_s
+        raise SqlNotResolved, "Telemetry::Query##{id} stores its SQL on its shape; load it through Query.with_sql" if text.empty? && group_hash
+        text
+      end
+
+      def timeline_label
+        sql.to_s.first(120)
+      end
+    end
+  end
+end

data/app/models/railwatch/telemetry/query_shape.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Telemetry
+    # One query group's normalized statement, stored once. The group hash is
+    # the gem's digest of the connection name and that statement, so a row
+    # whose own text digests to its group carries the statement and stores
+    # "" instead (Ingest::Writer); any other text stays on the row. See
+    # docs/architecture.md.
+    class QueryShape < TelemetryRecord
+      self.primary_key = :group_hash
+
+      BACKFILL_BATCH = 20_000
+
+      # Backfill for rows written before this table existed. Files a shape
+      # for each group whose earliest or newest row normalizes back to the
+      # group hash (a tenant's earliest rows may predate the gem sending
+      # normalized text; its newest will not), then blanks every row whose
+      # text is its group's shape, in id-range batches so no statement holds
+      # the tenant's write lock for long. Idempotent, and never blanks a row
+      # whose text is not on file. Returns what it did.
+      def self.dedupe_rows!(batch: BACKFILL_BATCH)
+        before = count
+        file_shapes!
+        moved = Query.where.not(sql: "")
+                     .where("queries.sql = (SELECT s.sql FROM query_shapes s WHERE s.group_hash = queries.group_hash)")
+                     .in_batches(of: batch, use_ranges: true).update_all(sql: "")
+        { shapes: count - before, rows: moved }
+      end
+
+      def self.file_shapes!
+        ends = "SELECT MIN(id) AS id FROM queries WHERE group_hash IS NOT NULL GROUP BY group_hash " \
+               "UNION SELECT MAX(id) FROM queries WHERE group_hash IS NOT NULL GROUP BY group_hash"
+        candidates = Query.joins("JOIN (#{ends}) ends ON ends.id = queries.id").where.not(sql: "")
+                          .pluck(:group_hash, :sql, :adapter, :connection)
+        shapes = candidates.filter_map do |group_hash, sql, adapter, connection|
+          digest, normalized = Railwatch::SqlNormalizer.group_and_normalized(sql, adapter: adapter, connection_name: connection)
+          { group_hash: group_hash, sql: normalized } if digest == group_hash
+        end
+        shapes.uniq { |shape| shape[:group_hash] }.each_slice(500) { |slice| insert_all(slice) }
+      end
+      private_class_method :file_shapes!
+    end
+  end
+end

data/app/models/railwatch/telemetry/release_health.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Telemetry
+    # Hourly release health per deploy, recomputed from raw `sessions` rows by
+    # ReleaseHealthRollupJob. The deploy *is* the release -- the gem stamps it
+    # on every record it ships -- so counting sessions per deploy is all this
+    # needs. Sentry's crash-free session rate, crash-free user rate, and
+    # adoption, over the same data.
+    #
+    # A session that spans two hours is counted in both buckets, like every
+    # other rollup here: the bucket, not the session, is the unit these numbers
+    # sum over. Every method must run inside environment.with_telemetry { }.
+    class ReleaseHealth < TelemetryRecord
+      self.table_name = "release_health"
+
+      scope :between, ->(from, to) { where(bucket: from.beginning_of_hour..to) }
+      # A nil deploy means "every release in the window", which is what the
+      # index page's headline numbers are.
+      scope :for_release, ->(deploy) { deploy ? where(deploy: deploy) : all }
+
+      TOTALS = %i[sessions sessions_errored sessions_crashed users users_crashed duration_sum duration_count].freeze
+
+      # Window totals for one release. Rates come back nil, not zero, when there
+      # is nothing to divide by, so an environment with no sessions yet never
+      # renders as "0% crash-free".
+      def self.for_deploy(deploy, from, to)
+        sessions, errored, crashed, users, users_crashed, duration_sum, duration_count =
+          between(from, to).for_release(deploy).pick(*TOTALS.map { |column| Arel.sql("COALESCE(SUM(#{column}), 0)") })
+        {
+          sessions: sessions, users: users,
+          crash_free_sessions: rate(sessions - crashed, sessions),
+          crash_free_users: rate(users - users_crashed, users),
+          errored: rate(errored, sessions),
+          avg_duration_ms: duration_count.zero? ? nil : (duration_sum / duration_count / 1000.0).round(2)
+        }
+      end
+
+      # Hourly points for the stacked sessions-by-status chart.
+      def self.series(deploy, from, to)
+        between(from, to).for_release(deploy).group(:bucket).order(:bucket)
+          .pluck(:bucket, Arel.sql("SUM(sessions)"), Arel.sql("SUM(sessions_errored)"), Arel.sql("SUM(sessions_crashed)"))
+          .map { |bucket, sessions, errored, crashed|
+            { t: bucket, sessions: sessions, ok: sessions - errored - crashed, errored: errored, crashed: crashed }
+          }
+      end
+
+      # Each release's share of the window's sessions, as a percentage -- how
+      # far a new release has rolled out.
+      def self.adoption(from, to)
+        counts = between(from, to).group(:deploy).sum(:sessions)
+        total = counts.values.sum
+        return counts.transform_values { 0.0 } if total.zero?
+        counts.transform_values { |n| (n * 100.0 / total).round(1) }
+      end
+
+      def self.rate(numerator, denominator)
+        return nil if denominator.to_i.zero?
+        (numerator * 100.0 / denominator).round(2)
+      end
+    end
+  end
+end

data/app/models/railwatch/telemetry/rollup.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Railwatch
+  require "tdigest"
+
+  module Telemetry
+    # Hourly aggregate per (record_type, group). Percentiles are exact within
+    # an hour and merged across hours with t-digest, so a 30-day p95 for a
+    # route is one query over 720 small rows.
+    class Rollup < TelemetryRecord
+      scope :for_type, ->(type) { where(record_type: type) }
+      scope :between, ->(from, to) { where(bucket: from.beginning_of_hour..to) }
+
+      def self.bucket_for(time)
+        time.utc.beginning_of_hour
+      end
+
+      # Merge a list of duration samples (microseconds) into the bucket.
+      def self.absorb!(record_type:, group_hash:, name:, bucket:, durations:, errors: 0, client_errors: 0, extra: {})
+        return if durations.empty?
+        row = find_or_initialize_by(record_type: record_type, group_hash: group_hash, bucket: bucket)
+        row.name = name
+        digest = row.digest ? TDigest::TDigest.from_bytes(row.digest) : TDigest::TDigest.new(0.01)
+        durations.each { |d| digest.push(d) }
+        digest.compress!
+        row.count += durations.size
+        row.error_count += errors
+        row.client_error_count += client_errors
+        row.duration_sum += durations.sum
+        row.duration_max = [ row.duration_max, durations.max ].max
+        row.p50 = digest.percentile(0.5).to_i
+        row.p95 = digest.percentile(0.95).to_i
+        row.p99 = digest.percentile(0.99).to_i
+        row.digest = digest.as_small_bytes
+        row.extra = row.extra.merge(extra) { |_k, a, b| a.is_a?(Numeric) && b.is_a?(Numeric) ? a + b : b }
+        row.save!
+        row
+      end
+
+      # The row absorb! would write over an empty bucket, as attributes for
+      # insert_all. RollupJob deletes an hour's rows and writes every group in
+      # one statement, so the read absorb! does per group is wasted there.
+      def self.fresh_attributes(record_type:, group_hash:, name:, bucket:, durations:, errors: 0, client_errors: 0, extra: {})
+        digest = TDigest::TDigest.new(0.01)
+        durations.each { |d| digest.push(d) }
+        digest.compress!
+        { record_type: record_type, group_hash: group_hash, name: name, bucket: bucket,
+         count: durations.size, error_count: errors, client_error_count: client_errors,
+         duration_sum: durations.sum, duration_max: durations.max,
+         p50: digest.percentile(0.5).to_i, p95: digest.percentile(0.95).to_i, p99: digest.percentile(0.99).to_i,
+         digest: digest.as_small_bytes, extra: extra }
+      end
+
+      # Aggregate a relation of rollups into one summary with merged percentiles.
+      def self.summarize(relation)
+        rows = relation.to_a
+        return { count: 0, errors: 0, client_errors: 0, avg: 0, p50: 0, p95: 0, p99: 0, max: 0 } if rows.empty?
+        # merge! pushes the row's centroids into one accumulating digest.
+        # `+` built a brand-new digest from both operands' centroids on every
+        # row, so merging N rows re-pushed every earlier centroid N times:
+        # 311 rows took 1.4s where merge! takes 0.5s, for the same percentiles.
+        merged = TDigest::TDigest.new(0.01)
+        rows.each { |r| merged.merge!(TDigest::TDigest.from_bytes(r.digest)) if r.digest }
+        count = rows.sum(&:count)
+        {
+          count: count,
+          errors: rows.sum(&:error_count),
+          client_errors: rows.sum(&:client_error_count),
+          avg: count.zero? ? 0 : (rows.sum(&:duration_sum) / count),
+          p50: merged.percentile(0.5).to_i,
+          p95: merged.percentile(0.95).to_i,
+          p99: merged.percentile(0.99).to_i,
+          max: rows.map(&:duration_max).max
+        }
+      end
+    end
+  end
+end

data/app/models/railwatch/telemetry/session.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Telemetry
+    # One session of the monitored application, for release health: a browser
+    # tab (reported by the gem's beacon client) or a server-side session (the
+    # gem's request middleware). The gem re-sends a live session on every flush
+    # interval, so the same session_id arrives many times and gets collapsed to
+    # one session per bucket by ReleaseHealthRollupJob.
+    class Session < TelemetryRecord
+      include Child
+
+      STATUSES = %w[started ok errored crashed].freeze
+
+      def crashed?
+        status == "crashed"
+      end
+
+      def timeline_label
+        session_id
+      end
+    end
+  end
+end

data/app/models/railwatch/telemetry/span.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Telemetry
+    # A custom span recorded with Railwatch.span("name") { }, a child of the
+    # request/job/command it ran inside.
+    class Span < TelemetryRecord
+      include Child
+
+      scope :failed, -> { where(status: "failed") }
+
+      # The wire record calls the user payload "attributes"; the column is
+      # `payload` because `attributes` is Active Record's own method.
+      def timeline_label
+        name.to_s.first(120)
+      end
+    end
+  end
+end

data/app/models/railwatch/telemetry/storage_op.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Telemetry
+    class StorageOp < TelemetryRecord
+      include Child
+
+      def timeline_label
+        key.to_s.first(120)
+      end
+    end
+  end
+end