railwatch 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2033d4d399f9d52afddba05178186c651c557d4a870decd78e0bf9cf90b6c9e7
-  data.tar.gz: 19cbbe8a3963391f28742b889fd5be1b68c006a59e2cefbbd0bcf2d33da1d3ca
+  metadata.gz: 35e3f727f70f3f5c73c0cbec4dfd83a79dc58141fd4b341c653433e8ae5be5d2
+  data.tar.gz: ff3832906ce67466c738a1ea9ec2d8178006c60958f1b3491e4fc31bea84aa3d
 SHA512:
-  metadata.gz: 43c847212862c33bf9bf9d6cb95354808ae3a033f79273351282c988a1334c83bb24f738789226f1feb9fc3798f60cba81bd460e2c783d8c1f8858ad1d4c1dc1
-  data.tar.gz: f73a3052b983d2493ca2a3e26a264110dc79ef37398badd3359fac7fe9f254f6eac3b6e3f34fc1583fb31434872f3e27faf9b31286cd7cb43544e5a3ba4514e3
+  metadata.gz: 674d67083f4ed5b3ca7153a18c0f6eaca9588763dc2d5ebf3fc7f40ce11ed54ed2626ea399186aa4d1acac4e22616775258ee3d76a6f01f3830aad11e5e422ad
+  data.tar.gz: 37db21eca95d40e29de0f47bf79b89e479acbd812d0381b808b606afe004efca328ac847b0a4db5a1bb78766df41fc44b9cf0e0738aa389e1aeead466ef441ef

data/CHANGELOG.md

@@ -6,6 +6,29 @@
      filled in by the release commit, which is also the only commit that
      touches lib/railwatch/version.rb and Gemfile.lock. See CONTRIBUTING.md. -->
 
+## 0.8.0 (2026-09-23)
+
+- Percentiles merged across hours are about ten times faster. Each
+  rollup's stored t-digest is read straight from its bytes and the
+  centroids are sorted once, instead of being pushed one at a time into
+  a new digest. A week of the platform's own query rollups (15,000 rows)
+  went from 1.6 s to 157 ms. Every page that shows a p50, p95 or p99
+  over a window reads this. The percentile is now nearest rank over the
+  centroids, the same rule the sub-hour charts use, so the two agree.
+- A new migration adds four indexes. Covering indexes serve the Jobs
+  page's per-queue breakdown and the Processes page's health series
+  (38 s and 19 s over a week of a 30 GB telemetry file, now 335 ms and
+  370 ms). There is an `occurred_at` index on broadcasts, the one
+  windowed table without one. A partial index covers executions that
+  carry an exception preview. The migration takes about 15 s on a
+  30 GB file.
+- Free-text search over jobs (and `Execution.named_like`, which MCP's
+  request search uses) matches names through the window's rollups and
+  fetches rows by `group_hash`, instead of reading every execution in
+  the window. A search matching nothing over a week took 657 ms and
+  takes 27 ms. A common text still takes the old path, which finds a
+  page within the first few thousand rows.
+
 ## 0.7.0 (2026-09-23)
 
 - The gem ships the dashboard's frontend source (`app/frontend`, without

data/app/models/railwatch/filter_query.rb

@@ -74,11 +74,12 @@ module Railwatch
       resource = resource.to_sym
       parsed = parse(query)
       fields = parsed[:fields].except(*Array(except).map(&:to_s))
-      scope = scope.where(occurred_at: bounded_time_range(fields, from, to))
+      range = bounded_time_range(fields, from, to)
+      scope = scope.where(occurred_at: range)
       scope = apply_common(scope, fields)
 
       case resource
-      when :jobs then apply_jobs(scope, fields, parsed[:text])
+      when :jobs then apply_jobs(scope, fields, parsed[:text], range)
       when :exceptions then apply_exceptions(scope, fields, parsed[:text])
       when :logs then apply_logs(scope, fields, parsed[:text])
       when :queries then apply_queries(scope, fields, parsed[:text])
@@ -120,15 +121,14 @@ module Railwatch
     end
     private_class_method :apply_common
 
-    def self.apply_jobs(scope, fields, text)
+    def self.apply_jobs(scope, fields, text, range)
       outcome = fields["outcome"].presence || fields["status"].presence
       scope = scope.where(queue: fields["queue"]) if fields["queue"].present?
       scope = scope.where(outcome: outcome) if outcome
       scope = scope.where(name: fields["class"]) if fields["class"].present?
       scope = scope.where(job_id: fields["job_id"]) if fields["job_id"].present?
       return scope unless text.present?
-      pattern = "%#{Telemetry::Execution.sanitize_sql_like(text)}%"
-      scope.where("name LIKE :pattern OR exception_preview LIKE :pattern", pattern: pattern)
+      scope.merge(Telemetry::Execution.named_like("job_attempt", text, range, previews: true))
     end
     private_class_method :apply_jobs
 

data/app/models/railwatch/telemetry/execution.rb

@@ -17,6 +17,53 @@ module Railwatch
       scope :between, ->(from, to) { where(occurred_at: from..to) }
       scope :failed, -> { where("status >= 500 OR outcome = 'failed'") }
 
+      # Executions of `kind` in [from, to] whose name contains `text`, or
+      # (with previews: true) whose exception preview does. A LIKE on the rows
+      # themselves reads every execution in the window until it has a page:
+      # 32 s for a week of the platform's own requests when the text is rare.
+      # Names come from a few hundred groups, so the text is matched against
+      # the window's rollups and the rows are fetched by group_hash. Rollups
+      # trail ingest by up to a minute, so the last FRESH_NAMES are matched on
+      # the rows themselves. Only failures carry a preview, and a partial
+      # index holds just those rows.
+      #
+      # Each branch names its index and the outer lookup is by rowid, so the
+      # plan does not depend on planner statistics (an embedded install's are
+      # sampled, and sampled statistics make kind look selective). When the
+      # text is common the old walk is the fast one -- a page of matches
+      # turns up in the first few thousand rows -- so a text the rollups say
+      # matches DENSE_MATCHES rows or more keeps it. On a copy of the
+      # platform's own tenant, a week of job attempts: a text matching
+      # nothing took 657 ms walking and 27 ms here; one matching 32,766 took
+      # 11 ms walking and 215 ms here.
+      FRESH_NAMES = 5.minutes
+      DENSE_MATCHES = 1_000
+
+      # `range` is used as given, so an exclusive or empty one (FilterQuery's
+      # after:/before: can narrow a window to nothing) stays that way.
+      def self.named_like(kind, text, range, previews: false)
+        from, to = range.begin, range.end
+        pattern = "%#{sanitize_sql_like(text)}%"
+        window = where(kind: kind, occurred_at: range)
+        groups = Rollup.for_type(kind).between(from, to).where("name LIKE ? ESCAPE '\\'", pattern)
+        if !TelemetryRecord.sqlite? || groups.sum(:count) >= DENSE_MATCHES
+          like = previews ? "name LIKE :pattern ESCAPE '\\' OR exception_preview LIKE :pattern ESCAPE '\\'" : "name LIKE :pattern ESCAPE '\\'"
+          return window.where(like, pattern: pattern)
+        end
+
+        by_group = indexed_by("index_executions_on_group_hash_and_occurred_at")
+          .where(group_hash: groups.distinct.select(:group_hash), occurred_at: range, kind: kind)
+        fresh = where(kind: kind, occurred_at: [ from, to - FRESH_NAMES ].max..to).where("name LIKE ? ESCAPE '\\'", pattern)
+        ids = [ by_group, fresh ]
+        ids << indexed_by("idx_executions_with_preview").where(kind: kind, occurred_at: range)
+          .where.not(exception_preview: nil).where("exception_preview LIKE ? ESCAPE '\\'", pattern) if previews
+        from("#{quoted_table_name} NOT INDEXED").where(kind: kind, occurred_at: range)
+          .where(ids.map { |branch| "#{quoted_table_name}.id IN (#{branch.select(:id).to_sql})" }.join(" OR "))
+      end
+
+      def self.indexed_by(index) = unscoped.from("#{quoted_table_name} INDEXED BY #{index}")
+      private_class_method :indexed_by
+
       # Servers that reported an execution since `time`. Asked once per kind:
       # there is no index led by occurred_at alone, so the obvious
       # `where(occurred_at: time..).distinct.pluck(:server)` scans the whole

data/app/models/railwatch/telemetry/rollup.rb

@@ -55,21 +55,16 @@ module Railwatch
       def self.summarize(relation)
         rows = relation.to_a
         return { count: 0, errors: 0, client_errors: 0, avg: 0, p50: 0, p95: 0, p99: 0, max: 0, extra: {} } if rows.empty?
-        # merge! pushes the row's centroids into one accumulating digest.
-        # `+` built a brand-new digest from both operands' centroids on every
-        # row, so merging N rows re-pushed every earlier centroid N times:
-        # 311 rows took 1.4s where merge! takes 0.5s, for the same percentiles.
-        merged = TDigest::TDigest.new(0.01)
-        rows.each { |r| merged.merge!(TDigest::TDigest.from_bytes(r.digest)) if r.digest }
+        p50, p95, p99 = merged_percentiles(rows, [ 0.5, 0.95, 0.99 ])
         count = rows.sum(&:count)
         {
           count: count,
           errors: rows.sum(&:error_count),
           client_errors: rows.sum(&:client_error_count),
           avg: count.zero? ? 0 : (rows.sum(&:duration_sum) / count),
-          p50: merged.percentile(0.5).to_i,
-          p95: merged.percentile(0.95).to_i,
-          p99: merged.percentile(0.99).to_i,
+          p50: p50,
+          p95: p95,
+          p99: p99,
           max: rows.map(&:duration_max).max,
           # Everything a type puts in `extra` -- llm_call's cost_nanos and
           # token counts, cache_event's hits and misses -- merged the way
@@ -80,6 +75,75 @@ module Railwatch
         }
       end
 
+      # Percentiles of the union of every row's centroids, read straight out
+      # of the stored bytes. Merging through TDigest pushed each centroid into
+      # a red-black tree one at a time: 2.8 s for a week of the platform's
+      # own query rollups (18,674 rows) and 7 s for thirty days, on every
+      # cold page load. Sorting the centroids once and walking their
+      # cumulative counts answers the same question without re-clustering:
+      # 353 ms and 1 s for those two windows. The answer is the first
+      # centroid whose running total reaches n * p, which is nearest rank --
+      # the rule raw_series uses for sub-hour buckets, so both readings of a
+      # window agree. (TDigest#percentile compares each centroid's midpoint
+      # instead, which puts the median of 1,000 sevens and 300 nines at 9.)
+      def self.merged_percentiles(rows, ps)
+        means = []
+        counts = []
+        rows.each { |row| centroids(row.digest, means, counts) if row.digest }
+        return ps.map { 0 } if means.empty?
+
+        order = (0...means.size).sort_by { |i| means[i] }
+        total = counts.sum
+        targets = ps.map { |p| total * p }
+        found = []
+        seen = 0
+        order.each do |i|
+          seen += counts[i]
+          found << means[i] while found.size < targets.size && seen >= targets[found.size]
+          break if found.size == targets.size
+        end
+        found << means[order.last] while found.size < targets.size
+        found.map(&:to_i)
+      end
+
+      # Appends one stored digest's centroid means and weights, in either of
+      # the tdigest gem's encodings (TDigest.from_bytes reads the same two).
+      def self.centroids(bytes, means, counts)
+        format, _compression, size = bytes.unpack("LdL")
+        case format
+        when TDigest::TDigest::VERBOSE_ENCODING
+          means.concat(bytes.unpack("@16d#{size}"))
+          counts.concat(bytes.unpack("@#{16 + 8 * size}L#{size}"))
+        when TDigest::TDigest::SMALL_ENCODING
+          # Means are delta-encoded 4-byte floats; weights are 7-bit varints,
+          # one byte each unless a centroid holds 128 or more samples.
+          mean = 0.0
+          bytes.unpack("@16f#{size}").each { |delta| means << (mean += delta) }
+          weights = bytes.byteslice(16 + 4 * size, bytes.bytesize).unpack("C*")
+          if weights.size == size
+            counts.concat(weights)
+          else
+            at = 0
+            size.times do
+              byte = weights[at]
+              at += 1
+              weight = byte & 0x7f
+              shift = 7
+              while byte & 0x80 != 0
+                byte = weights[at] || 0
+                at += 1
+                weight += (byte & 0x7f) << shift
+                shift += 7
+              end
+              counts << weight
+            end
+          end
+        else
+          raise ArgumentError, "unknown t-digest encoding #{format}"
+        end
+      end
+      private_class_method :centroids
+
       def self.merge_extras(rows)
         rows.each_with_object({}) do |row, out|
           row.extra.each do |key, value|

data/db/railwatch_telemetry_migrate/20260922000000_add_covering_indexes_for_dashboard_aggregates.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+# Two dashboard aggregates read every row in the window and only a few
+# narrow columns of each: the Jobs page's per-queue breakdown and the
+# Processes page's health series. Through (kind, occurred_at) and
+# (sampled_at) SQLite finds the rows in order but then fetches each one
+# from the table, and on a telemetry file much larger than memory those
+# rows are scattered pages on disk. On the platform's own 30 GB tenant,
+# over a week: 38 s for the queue breakdown (491,000 job attempts) and
+# 19 s for the health series (263,000 samples), with the GVL held the
+# whole time, so every other request in the process waited too.
+#
+# Both indexes carry every column their query reads, so the whole answer
+# comes from one contiguous index range: 335 ms and 370 ms on a copy of
+# the same file. They cost 151 MB and 29 MB there, and took 13 s and 2 s
+# to build.
+#
+# broadcasts is the one windowed table with no index led by occurred_at,
+# so the Broadcasts page's "recent" list sorted the whole window to show
+# 100 rows. Every other raw table has had this index since it was created.
+#
+# Free-text search over jobs and requests also matches the exception
+# preview, which only a failed execution carries (15 of 491,000 job
+# attempts in that same week). A partial index over just those rows lets
+# Execution.named_like seek them instead of reading the window.
+class AddCoveringIndexesForDashboardAggregates < ActiveRecord::Migration[8.1]
+  def change
+    add_index :executions, [ :kind, :occurred_at, :queue, :outcome, :queue_latency ], name: "idx_executions_queue_stats"
+    add_index :health_samples, [ :sampled_at, :threads_max, :threads_busy, :backlog, :queue_depth, :queue_latency ],
+              name: "idx_health_samples_series"
+    add_index :broadcasts, :occurred_at
+    add_index :executions, [ :kind, :occurred_at ], name: "idx_executions_with_preview", where: "exception_preview IS NOT NULL"
+  end
+end

data/lib/railwatch/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Railwatch
-  VERSION = "0.7.0"
+  VERSION = "0.8.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: railwatch
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Cole Robertson
@@ -403,6 +403,7 @@ files:
 - db/railwatch_telemetry_migrate/20260915210000_add_slowest_index_to_queries.rb
 - db/railwatch_telemetry_migrate/20260917010000_add_batch_ledger_to_ingest_batches.rb
 - db/railwatch_telemetry_migrate/20260919000100_create_export_queue.rb
+- db/railwatch_telemetry_migrate/20260922000000_add_covering_indexes_for_dashboard_aggregates.rb
 - docs/ai-and-mcp.md
 - docs/configuration.md
 - docs/embedded.md