railwatch 0.7.0 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2033d4d399f9d52afddba05178186c651c557d4a870decd78e0bf9cf90b6c9e7
-  data.tar.gz: 19cbbe8a3963391f28742b889fd5be1b68c006a59e2cefbbd0bcf2d33da1d3ca
+  metadata.gz: c621aa5be8712487e490453e9b62ab5d3f75666b951270d50acf77b00ad03a29
+  data.tar.gz: 5464f54bb6968924b1155456dcd22c5dada112980760340beb74632a169d5791
 SHA512:
-  metadata.gz: 43c847212862c33bf9bf9d6cb95354808ae3a033f79273351282c988a1334c83bb24f738789226f1feb9fc3798f60cba81bd460e2c783d8c1f8858ad1d4c1dc1
-  data.tar.gz: f73a3052b983d2493ca2a3e26a264110dc79ef37398badd3359fac7fe9f254f6eac3b6e3f34fc1583fb31434872f3e27faf9b31286cd7cb43544e5a3ba4514e3
+  metadata.gz: 035642f4a96ae1803f01079f618315a7c1ff54657f6111f52071ae32896782118a40e3f7075ddfb60910625899e589c5cea8f5b3b7c999af42211d097eb2e161
+  data.tar.gz: 4dc2f2217692fa974f06c7c2c7b487896206a0cefccd7301f9802d3eb192dc47900bdfdf96a9a07809d86b41c6a59292fc0df27382b7636167331fcb59c02978

data/CHANGELOG.md

@@ -6,6 +6,41 @@
      filled in by the release commit, which is also the only commit that
      touches lib/railwatch/version.rb and Gemfile.lock. See CONTRIBUTING.md. -->
 
+## 0.8.1 (2026-09-23)
+
+- The Tenants page reads only through the tenant-led indexes. Sparklines
+  are drawn for the tenants already found, by name. The untagged share is
+  the window's requests less the tagged ones, and the tagged ones are
+  counted per tenant. Both used to read every request in the window: over
+  a week, 11 s on an app with no tenants and 24 s on one with them.
+- A partial covering index serves the People page: 4 s over 30 days,
+  now 18 ms, from under a megabyte.
+- The Profiles page takes the window's execution count from rollups
+  instead of counting every execution (1.2 s over a week).
+
+## 0.8.0 (2026-09-23)
+
+- Percentiles merged across hours are about ten times faster. Each
+  rollup's stored t-digest is read straight from its bytes and the
+  centroids are sorted once, instead of being pushed one at a time into
+  a new digest. A week of the platform's own query rollups (15,000 rows)
+  went from 1.6 s to 157 ms. Every page that shows a p50, p95 or p99
+  over a window reads this. The percentile is now nearest rank over the
+  centroids, the same rule the sub-hour charts use, so the two agree.
+- A new migration adds four indexes. Covering indexes serve the Jobs
+  page's per-queue breakdown and the Processes page's health series
+  (38 s and 19 s over a week of a 30 GB telemetry file, now 335 ms and
+  370 ms). There is an `occurred_at` index on broadcasts, the one
+  windowed table without one. A partial index covers executions that
+  carry an exception preview. The migration takes about 15 s on a
+  30 GB file.
+- Free-text search over jobs (and `Execution.named_like`, which MCP's
+  request search uses) matches names through the window's rollups and
+  fetches rows by `group_hash`, instead of reading every execution in
+  the window. A search matching nothing over a week took 657 ms and
+  takes 27 ms. A common text still takes the old path, which finds a
+  page within the first few thousand rows.
+
 ## 0.7.0 (2026-09-23)
 
 - The gem ships the dashboard's frontend source (`app/frontend`, without

data/app/controllers/railwatch/profiles_controller.rb

@@ -13,7 +13,7 @@ module Railwatch
       rows, count, avg_samples, profiled, executions = telemetry do
         scope = Telemetry::Profile.between(from, to)
         [ scope.recent.limit(SCAN_LIMIT).select(:id, :group_hash, :execution_preview, :duration, :samples, :occurred_at).to_a,
-          scope.count, scope.average(:samples), scope.distinct.count(:execution_id), Telemetry::Execution.between(from, to).count ]
+          scope.count, scope.average(:samples), scope.distinct.count(:execution_id), executions_in(from, to) ]
       end
       render inertia: { profiles: groups(rows),
                         summary: { profiles: count, executions: executions, profiled: profiled, avg_samples: avg_samples.to_f.round(0) } }
@@ -29,6 +29,19 @@ module Railwatch
 
     private
 
+    # How many executions the window holds, as the profiled share's
+    # denominator. Every kind is rolled up, so the whole hours come from a
+    # few hundred rollup rows rather than counting every execution (1.2 s
+    # over a week of the platform's own); only the part of the first hour
+    # inside the window is counted from the rows.
+    def executions_in(from, to)
+      whole = from.beginning_of_hour == from ? from : from.beginning_of_hour + 1.hour
+      return Telemetry::Execution.between(from, to).count if whole >= to
+
+      Telemetry::Execution.where(occurred_at: from...whole).count +
+        Telemetry::Rollup.for_type(Telemetry::Execution::KINDS).between(whole, to).sum(:count)
+    end
+
     # Rows arrive newest-first, so the head of each group is the profile the
     # page links to.
     def groups(rows)

data/app/models/railwatch/filter_query.rb

@@ -74,11 +74,12 @@ module Railwatch
       resource = resource.to_sym
       parsed = parse(query)
       fields = parsed[:fields].except(*Array(except).map(&:to_s))
-      scope = scope.where(occurred_at: bounded_time_range(fields, from, to))
+      range = bounded_time_range(fields, from, to)
+      scope = scope.where(occurred_at: range)
       scope = apply_common(scope, fields)
 
       case resource
-      when :jobs then apply_jobs(scope, fields, parsed[:text])
+      when :jobs then apply_jobs(scope, fields, parsed[:text], range)
       when :exceptions then apply_exceptions(scope, fields, parsed[:text])
       when :logs then apply_logs(scope, fields, parsed[:text])
       when :queries then apply_queries(scope, fields, parsed[:text])
@@ -120,15 +121,14 @@ module Railwatch
     end
     private_class_method :apply_common
 
-    def self.apply_jobs(scope, fields, text)
+    def self.apply_jobs(scope, fields, text, range)
       outcome = fields["outcome"].presence || fields["status"].presence
       scope = scope.where(queue: fields["queue"]) if fields["queue"].present?
       scope = scope.where(outcome: outcome) if outcome
       scope = scope.where(name: fields["class"]) if fields["class"].present?
       scope = scope.where(job_id: fields["job_id"]) if fields["job_id"].present?
       return scope unless text.present?
-      pattern = "%#{Telemetry::Execution.sanitize_sql_like(text)}%"
-      scope.where("name LIKE :pattern OR exception_preview LIKE :pattern", pattern: pattern)
+      scope.merge(Telemetry::Execution.named_like("job_attempt", text, range, previews: true))
     end
     private_class_method :apply_jobs
 

data/app/models/railwatch/telemetry/execution.rb

@@ -17,6 +17,53 @@ module Railwatch
       scope :between, ->(from, to) { where(occurred_at: from..to) }
       scope :failed, -> { where("status >= 500 OR outcome = 'failed'") }
 
+      # Executions of `kind` in [from, to] whose name contains `text`, or
+      # (with previews: true) whose exception preview does. A LIKE on the rows
+      # themselves reads every execution in the window until it has a page:
+      # 32 s for a week of the platform's own requests when the text is rare.
+      # Names come from a few hundred groups, so the text is matched against
+      # the window's rollups and the rows are fetched by group_hash. Rollups
+      # trail ingest by up to a minute, so the last FRESH_NAMES are matched on
+      # the rows themselves. Only failures carry a preview, and a partial
+      # index holds just those rows.
+      #
+      # Each branch names its index and the outer lookup is by rowid, so the
+      # plan does not depend on planner statistics (an embedded install's are
+      # sampled, and sampled statistics make kind look selective). When the
+      # text is common the old walk is the fast one -- a page of matches
+      # turns up in the first few thousand rows -- so a text the rollups say
+      # matches DENSE_MATCHES rows or more keeps it. On a copy of the
+      # platform's own tenant, a week of job attempts: a text matching
+      # nothing took 657 ms walking and 27 ms here; one matching 32,766 took
+      # 11 ms walking and 215 ms here.
+      FRESH_NAMES = 5.minutes
+      DENSE_MATCHES = 1_000
+
+      # `range` is used as given, so an exclusive or empty one (FilterQuery's
+      # after:/before: can narrow a window to nothing) stays that way.
+      def self.named_like(kind, text, range, previews: false)
+        from, to = range.begin, range.end
+        pattern = "%#{sanitize_sql_like(text)}%"
+        window = where(kind: kind, occurred_at: range)
+        groups = Rollup.for_type(kind).between(from, to).where("name LIKE ? ESCAPE '\\'", pattern)
+        if !TelemetryRecord.sqlite? || groups.sum(:count) >= DENSE_MATCHES
+          like = previews ? "name LIKE :pattern ESCAPE '\\' OR exception_preview LIKE :pattern ESCAPE '\\'" : "name LIKE :pattern ESCAPE '\\'"
+          return window.where(like, pattern: pattern)
+        end
+
+        by_group = indexed_by("index_executions_on_group_hash_and_occurred_at")
+          .where(group_hash: groups.distinct.select(:group_hash), occurred_at: range, kind: kind)
+        fresh = where(kind: kind, occurred_at: [ from, to - FRESH_NAMES ].max..to).where("name LIKE ? ESCAPE '\\'", pattern)
+        ids = [ by_group, fresh ]
+        ids << indexed_by("idx_executions_with_preview").where(kind: kind, occurred_at: range)
+          .where.not(exception_preview: nil).where("exception_preview LIKE ? ESCAPE '\\'", pattern) if previews
+        from("#{quoted_table_name} NOT INDEXED").where(kind: kind, occurred_at: range)
+          .where(ids.map { |branch| "#{quoted_table_name}.id IN (#{branch.select(:id).to_sql})" }.join(" OR "))
+      end
+
+      def self.indexed_by(index) = unscoped.from("#{quoted_table_name} INDEXED BY #{index}")
+      private_class_method :indexed_by
+
       # Servers that reported an execution since `time`. Asked once per kind:
       # there is no index led by occurred_at alone, so the obvious
       # `where(occurred_at: time..).distinct.pluck(:server)` scans the whole

data/app/models/railwatch/telemetry/rollup.rb

@@ -55,21 +55,16 @@ module Railwatch
       def self.summarize(relation)
         rows = relation.to_a
         return { count: 0, errors: 0, client_errors: 0, avg: 0, p50: 0, p95: 0, p99: 0, max: 0, extra: {} } if rows.empty?
-        # merge! pushes the row's centroids into one accumulating digest.
-        # `+` built a brand-new digest from both operands' centroids on every
-        # row, so merging N rows re-pushed every earlier centroid N times:
-        # 311 rows took 1.4s where merge! takes 0.5s, for the same percentiles.
-        merged = TDigest::TDigest.new(0.01)
-        rows.each { |r| merged.merge!(TDigest::TDigest.from_bytes(r.digest)) if r.digest }
+        p50, p95, p99 = merged_percentiles(rows, [ 0.5, 0.95, 0.99 ])
         count = rows.sum(&:count)
         {
           count: count,
           errors: rows.sum(&:error_count),
           client_errors: rows.sum(&:client_error_count),
           avg: count.zero? ? 0 : (rows.sum(&:duration_sum) / count),
-          p50: merged.percentile(0.5).to_i,
-          p95: merged.percentile(0.95).to_i,
-          p99: merged.percentile(0.99).to_i,
+          p50: p50,
+          p95: p95,
+          p99: p99,
           max: rows.map(&:duration_max).max,
           # Everything a type puts in `extra` -- llm_call's cost_nanos and
           # token counts, cache_event's hits and misses -- merged the way
@@ -80,6 +75,75 @@ module Railwatch
         }
       end
 
+      # Percentiles of the union of every row's centroids, read straight out
+      # of the stored bytes. Merging through TDigest pushed each centroid into
+      # a red-black tree one at a time: 2.8 s for a week of the platform's
+      # own query rollups (18,674 rows) and 7 s for thirty days, on every
+      # cold page load. Sorting the centroids once and walking their
+      # cumulative counts answers the same question without re-clustering:
+      # 353 ms and 1 s for those two windows. The answer is the first
+      # centroid whose running total reaches n * p, which is nearest rank --
+      # the rule raw_series uses for sub-hour buckets, so both readings of a
+      # window agree. (TDigest#percentile compares each centroid's midpoint
+      # instead, which puts the median of 1,000 sevens and 300 nines at 9.)
+      def self.merged_percentiles(rows, ps)
+        means = []
+        counts = []
+        rows.each { |row| centroids(row.digest, means, counts) if row.digest }
+        return ps.map { 0 } if means.empty?
+
+        order = (0...means.size).sort_by { |i| means[i] }
+        total = counts.sum
+        targets = ps.map { |p| total * p }
+        found = []
+        seen = 0
+        order.each do |i|
+          seen += counts[i]
+          found << means[i] while found.size < targets.size && seen >= targets[found.size]
+          break if found.size == targets.size
+        end
+        found << means[order.last] while found.size < targets.size
+        found.map(&:to_i)
+      end
+
+      # Appends one stored digest's centroid means and weights, in either of
+      # the tdigest gem's encodings (TDigest.from_bytes reads the same two).
+      def self.centroids(bytes, means, counts)
+        format, _compression, size = bytes.unpack("LdL")
+        case format
+        when TDigest::TDigest::VERBOSE_ENCODING
+          means.concat(bytes.unpack("@16d#{size}"))
+          counts.concat(bytes.unpack("@#{16 + 8 * size}L#{size}"))
+        when TDigest::TDigest::SMALL_ENCODING
+          # Means are delta-encoded 4-byte floats; weights are 7-bit varints,
+          # one byte each unless a centroid holds 128 or more samples.
+          mean = 0.0
+          bytes.unpack("@16f#{size}").each { |delta| means << (mean += delta) }
+          weights = bytes.byteslice(16 + 4 * size, bytes.bytesize).unpack("C*")
+          if weights.size == size
+            counts.concat(weights)
+          else
+            at = 0
+            size.times do
+              byte = weights[at]
+              at += 1
+              weight = byte & 0x7f
+              shift = 7
+              while byte & 0x80 != 0
+                byte = weights[at] || 0
+                at += 1
+                weight += (byte & 0x7f) << shift
+                shift += 7
+              end
+              counts << weight
+            end
+          end
+        else
+          raise ArgumentError, "unknown t-digest encoding #{format}"
+        end
+      end
+      private_class_method :centroids
+
       def self.merge_extras(rows)
         rows.each_with_object({}) do |row, out|
           row.extra.each do |key, value|

data/app/models/railwatch/telemetry/tenant.rb

@@ -18,6 +18,9 @@ module Railwatch
       # aggregate), so only the busiest tenants get one; the rest report 0.
       P95_TENANTS = 50
       SPARKLINE_BUCKETS = 12
+      # Tenant names are bound one variable each, and SQLite allows 32,766
+      # per statement; an app with more tenants than that is asked in slices.
+      TENANT_SLICE = 1_000
       SORTS = { "requests" => :requests, "errors" => :errors, "p95" => :p95, "users" => :users }.freeze
 
       ERRORS_SQL = Arel.sql("SUM(CASE WHEN status >= 500 THEN 1 ELSE 0 END)")
@@ -35,7 +38,7 @@ module Railwatch
         absorb_counts(rows, filtered(Telemetry::Exception.between(from, to), q), :exceptions)
         absorb_counts(rows, filtered(Telemetry::Log.between(from, to), q), :logs)
         absorb_last_seen(rows, from, to, q)
-        absorb_sparklines(rows, from, to, q)
+        absorb_sparklines(rows, from, to)
         absorb_p95s(rows.values.max_by(P95_TENANTS) { |r| r[:requests] }, from, to)
         sorted(rows.values, sort, dir)
       end
@@ -43,20 +46,31 @@ module Railwatch
       # Headline numbers for the index page, computed from the rows it shows
       # (so the top-tenant share is a share of the listed tenants' requests)
       # plus the share of requests in the window that carry no tenant at all.
+      #
+      # The untagged count is the window's requests less the tagged ones,
+      # and the tagged ones are counted per tenant in app_tenant's index.
+      # Counting either side with app_tenant IS [NOT] NULL read every request
+      # in the window instead: 1 s on an untagged app, 24 s on a tagged one.
       def self.overview(rows, from, to)
         requests = Telemetry::Execution.requests.between(from, to)
         total = requests.count
-        untagged = requests.where(app_tenant: nil).count
+        tagged_total = tagged_tenants(from, to).each_slice(TENANT_SLICE).sum { |slice| requests.where(app_tenant: slice).count }
         tagged = rows.sum { |r| r[:requests] }
         top = rows.max_by { |r| r[:requests] }
         {
           tenants: rows.size, top_tenant: top && top[:tenant],
           top_share: tagged.zero? ? 0.0 : (top[:requests] * 100.0 / tagged).round(1),
           with_errors: rows.count { |r| r[:errors].positive? },
-          untagged_share: total.zero? ? 0.0 : (untagged * 100.0 / total).round(1)
+          untagged_share: total.zero? ? 0.0 : ((total - tagged_total) * 100.0 / total).round(1)
         }
       end
 
+      # Every tenant that sent anything in the window, from the tenant-led
+      # index (a skip-scan over distinct values, not the window's rows).
+      def self.tagged_tenants(from, to)
+        Telemetry::Execution.between(from, to).where.not(app_tenant: nil).distinct.pluck(:app_tenant)
+      end
+
       # Window totals for one tenant. Durations in milliseconds, like the rest
       # of the tenant props.
       def self.summary(tenant, from, to)
@@ -129,9 +143,19 @@ module Railwatch
           .each { |tenant, at| rows[tenant][:last_seen_at] = at }
       end
 
-      def self.absorb_sparklines(rows, from, to, q)
+      # Only for the tenants already found, by name: grouped by tenant and a
+      # time expression SQLite would not seek the app_tenant index and read
+      # every request in the window instead -- 11 s over a week on an app
+      # with no tenants at all, to draw nothing.
+      def self.absorb_sparklines(rows, from, to)
+        tenants = rows.keys.compact
+        return if tenants.empty?
+
         width = bucket_width(from, to, SPARKLINE_BUCKETS)
-        counts = filtered(Telemetry::Execution.requests.between(from, to), q).group(:app_tenant, bucket_sql(from, width)).count
+        counts = tenants.each_slice(TENANT_SLICE).flat_map do |slice|
+          Telemetry::Execution.requests.between(from, to).where(app_tenant: slice)
+            .group(:app_tenant, bucket_sql(from, width)).count.to_a
+        end
         counts.each do |(tenant, index), count|
           rows[tenant][:sparkline][[ index.to_i, SPARKLINE_BUCKETS - 1 ].min] += count
         end

data/db/railwatch_telemetry_migrate/20260922000000_add_covering_indexes_for_dashboard_aggregates.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+# Two dashboard aggregates read every row in the window and only a few
+# narrow columns of each: the Jobs page's per-queue breakdown and the
+# Processes page's health series. Through (kind, occurred_at) and
+# (sampled_at) SQLite finds the rows in order but then fetches each one
+# from the table, and on a telemetry file much larger than memory those
+# rows are scattered pages on disk. On the platform's own 30 GB tenant,
+# over a week: 38 s for the queue breakdown (491,000 job attempts) and
+# 19 s for the health series (263,000 samples), with the GVL held the
+# whole time, so every other request in the process waited too.
+#
+# Both indexes carry every column their query reads, so the whole answer
+# comes from one contiguous index range: 335 ms and 370 ms on a copy of
+# the same file. They cost 151 MB and 29 MB there, and took 13 s and 2 s
+# to build.
+#
+# broadcasts is the one windowed table with no index led by occurred_at,
+# so the Broadcasts page's "recent" list sorted the whole window to show
+# 100 rows. Every other raw table has had this index since it was created.
+#
+# Free-text search over jobs and requests also matches the exception
+# preview, which only a failed execution carries (15 of 491,000 job
+# attempts in that same week). A partial index over just those rows lets
+# Execution.named_like seek them instead of reading the window.
+class AddCoveringIndexesForDashboardAggregates < ActiveRecord::Migration[8.1]
+  def change
+    add_index :executions, [ :kind, :occurred_at, :queue, :outcome, :queue_latency ], name: "idx_executions_queue_stats"
+    add_index :health_samples, [ :sampled_at, :threads_max, :threads_busy, :backlog, :queue_depth, :queue_latency ],
+              name: "idx_health_samples_series"
+    add_index :broadcasts, :occurred_at
+    add_index :executions, [ :kind, :occurred_at ], name: "idx_executions_with_preview", where: "exception_preview IS NOT NULL"
+  end
+end

data/db/railwatch_telemetry_migrate/20260923000000_add_people_index.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# The People page groups the window's signed-in requests by user_ref.
+# Through (user_ref, occurred_at) SQLite found those rows but fetched each
+# one from the table for kind and status: 4 s over 30 days on the
+# platform's own tenant, where 18,000 of 2.8 million executions carry a
+# user. This index holds every column the query reads, and only for rows
+# that have a user, so it answers in 18 ms from under a megabyte. It took
+# 10 s to build on a copy of that 30 GB file.
+class AddPeopleIndex < ActiveRecord::Migration[8.1]
+  def change
+    add_index :executions, [ :user_ref, :kind, :occurred_at, :status ], name: "idx_executions_people",
+              where: "user_ref IS NOT NULL"
+  end
+end

data/lib/railwatch/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Railwatch
-  VERSION = "0.7.0"
+  VERSION = "0.8.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: railwatch
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.1
 platform: ruby
 authors:
 - Cole Robertson
@@ -403,6 +403,8 @@ files:
 - db/railwatch_telemetry_migrate/20260915210000_add_slowest_index_to_queries.rb
 - db/railwatch_telemetry_migrate/20260917010000_add_batch_ledger_to_ingest_batches.rb
 - db/railwatch_telemetry_migrate/20260919000100_create_export_queue.rb
+- db/railwatch_telemetry_migrate/20260922000000_add_covering_indexes_for_dashboard_aggregates.rb
+- db/railwatch_telemetry_migrate/20260923000000_add_people_index.rb
 - docs/ai-and-mcp.md
 - docs/configuration.md
 - docs/embedded.md