railwatch 0.3.7 → 0.5.0

data/app/jobs/railwatch/detect_performance_issues_job.rb

@@ -12,7 +12,8 @@ module Railwatch
     MAX_ROLLUP_ROWS_PER_RULE = MAX_GROUPS_PER_RULE * 25
 
     TYPE_FOR = { "requests" => "request", "jobs" => "job_attempt", "commands" => "command", "queries" => "query",
-                 "scheduled_tasks" => "scheduled_task", "outgoing_requests" => "outgoing_request" }.freeze
+                 "scheduled_tasks" => "scheduled_task", "outgoing_requests" => "outgoing_request",
+                 "llm_calls" => "llm_call", "llm_tools" => "llm_tool" }.freeze
 
     def perform(environment)
       now = Time.current
@@ -51,19 +52,28 @@ module Railwatch
           .transform_values { |group_rows| [ group_rows.first.name, Telemetry::Rollup.summarize(group_rows) ] }
     end
 
+    NANOS_PER_DOLLAR = 1_000_000_000.0
+
     def value_for(metric, s)
+      extra = s[:extra] || {}
       case metric
       when "p95" then s[:p95] / 1000.0
       when "max" then s[:max] / 1000.0
       when "avg" then s[:avg] / 1000.0
       when "error_rate", "failure_rate" then s[:count].zero? ? nil : (s[:errors] * 100.0 / s[:count])
+      # Money and tokens are sums over the window, not percentiles of it.
+      when "spend" then extra["cost_nanos"].to_i / NANOS_PER_DOLLAR
+      when "tokens" then extra["input_tokens"].to_i + extra["output_tokens"].to_i
+      # A cut-off answer is a successful call, so it is invisible to
+      # error_rate. This is the only way to be told about it.
+      when "truncation_rate" then s[:count].zero? ? nil : (extra["truncated"].to_i * 100.0 / s[:count])
       end
     end
 
     def open_issue(environment, threshold, group_hash, name, value, from, now)
       issue, outcome = Issue.record_occurrence!(
         environment: environment, group_hash: "perf:#{threshold.id}:#{group_hash}", kind: "performance",
-        title: "#{name} exceeded #{threshold.metric} #{threshold.limit}#{threshold.metric.end_with?('rate') ? '%' : 'ms'} (#{value.round(1)})",
+        title: "#{name} exceeded #{threshold.metric} #{threshold.format_value(threshold.limit)} (#{threshold.format_value(value.round(2))})",
         culprit: name, occurred_at: now, deploy: nil, user_ref: nil,
         sample: { threshold_id: threshold.id, value: value.round(2), metric: threshold.metric, limit: threshold.limit })
       carry_detection(issue, outcome) do

data/app/models/railwatch/anomaly_rule.rb

@@ -10,13 +10,22 @@ module Railwatch
     MAX_BASELINE_DAYS = 30
     MAX_DEVIATION = 10
     MAX_WINDOW_MINUTES = 1_440
-    TARGET_KINDS = %w[requests jobs queries outgoing_requests scheduled_tasks].freeze
-    METRICS = %w[p95 avg count error_rate].freeze
+    TARGET_KINDS = %w[requests jobs queries outgoing_requests scheduled_tasks llm_calls].freeze
+    # spend included because "we are spending well above baseline for this
+    # hour of the week" is the LLM alert that needs no threshold picked.
+    METRICS = %w[p95 avg count error_rate spend].freeze
+    SHARED_METRICS = %w[p95 avg count error_rate].freeze
+    LLM_METRICS = %w[spend].freeze
+
+    def self.metrics_for(target_kind)
+      target_kind == "llm_calls" ? SHARED_METRICS + LLM_METRICS : SHARED_METRICS
+    end
 
     def environment = Environment.current
 
     validates :target_kind, inclusion: { in: TARGET_KINDS }
     validates :metric, inclusion: { in: METRICS }
+    validate :metric_applies_to_target_kind
     validates :target, presence: true, length: { maximum: 256 }
     validates :deviation, numericality: { greater_than: 0, less_than_or_equal_to: MAX_DEVIATION }
     validates :window_minutes, numericality: { only_integer: true, greater_than: 0,
@@ -29,5 +38,14 @@ module Railwatch
     def description
       "#{target_kind} #{target == '*' ? 'all' : target}: #{metric} > #{deviation}σ above #{baseline_days}-day baseline"
     end
+
+    private
+
+    def metric_applies_to_target_kind
+      return if metric.blank? || target_kind.blank?
+      return if AnomalyRule.metrics_for(target_kind).include?(metric)
+
+      errors.add(:metric, "is not available for #{target_kind}")
+    end
   end
 end

data/app/models/railwatch/ingest/batch.rb

@@ -182,6 +182,7 @@ module Railwatch
             # batch does not need.
             raise TypeError, "record must be an object" unless rec.is_a?(Hash)
             raise TypeError, "t must be a string" unless rec["t"].is_a?(String)
+            Ingest::Mapper.validate_version!(rec)
           else
             Ingest::Mapper.validate_record!(rec)
           end

data/app/models/railwatch/ingest/mapper.rb

@@ -111,6 +111,7 @@ module Railwatch
       def validate_record!(rec, identifiers: true)
         raise TypeError, "record must be an object" unless rec.is_a?(Hash)
         raise TypeError, "t must be a string" unless rec["t"].is_a?(String)
+        validate_version!(rec)
 
         if identifiers
           validate_identifier!(rec, "_group", GROUP_HASH, "32 hexadecimal characters")
@@ -129,6 +130,17 @@ module Railwatch
         validate_locals!(rec["locals"])
       end
 
+      # A record shape is named by its type and version (Railwatch::Record::VERSIONS).
+      # This mapper only knows the versions the gem it ships with emits; a
+      # record from a newer or older gem is refused by name rather than read
+      # through columns that may no longer mean the same thing.
+      def validate_version!(rec)
+        expected = Record::VERSIONS[rec["t"].to_sym]
+        return if expected.nil? # unknown type: row_for answers nil and the batch rejects it as such
+
+        raise TypeError, "#{rec["t"]} v#{rec["v"].inspect} is not v#{expected}" unless rec["v"].is_a?(Integer) && rec["v"] == expected
+      end
+
       def validate_identifier!(rec, field, pattern, description)
         value = rec[field]
         return if value.nil?

data/app/models/railwatch/telemetry/aggregations.rb

@@ -9,18 +9,154 @@ module Railwatch
     class Aggregations
       GROUPED_SORT_FIELDS = %i[count p50 p95 p99 errors avg max].freeze
 
-      # Time-bucketed series from rollups for charts: [{t, count, errors, p50, p95, p99}]
-      def self.series(environment, record_type, from:, to:, group_hash: nil, name: nil)
-        environment.with_telemetry do
-          scope = Telemetry::Rollup.for_type(record_type).between(from, to)
-          scope = scope.where(group_hash: group_hash) if group_hash
-          scope = scope.where(name: name) if name
-          scope.group(:bucket).order(:bucket)
-               .pluck(:bucket, Arel.sql("SUM(count)"), Arel.sql("SUM(error_count)"), Arel.sql("SUM(client_error_count)"), Arel.sql("SUM(duration_sum)"), Arel.sql("MAX(p50)"), Arel.sql("MAX(p95)"), Arel.sql("MAX(p99)"))
-               .map { |b, c, e, ce, ds, p50, p95, p99| { t: b, count: c, errors: e, client_errors: ce, avg: c.zero? ? 0 : ds / c / 1000.0, p50: p50 / 1000.0, p95: p95 / 1000.0, p99: p99 / 1000.0 } }
+      # Bucket widths a chart can be drawn at. Anything under an hour is read
+      # from the raw tables (rollups are hourly); an hour and up groups rollups.
+      STEPS = { "1m" => 1.minute, "5m" => 5.minutes, "15m" => 15.minutes, "1h" => 1.hour, "6h" => 6.hours, "1d" => 1.day }.freeze
+      # A step is offered for a window when it draws at least two points and at
+      # most this many: more bars than pixels is mush, and a sub-hour step over
+      # a long window is also a long raw scan.
+      MAX_POINTS = 400
+
+      # The bucket a window is drawn at unless the page asks for another one.
+      # Sub-hour steps scan raw rows, so they are the default only where that
+      # scan is a few thousand rows; a day and beyond stays on rollups.
+      def self.default_step(from, to)
+        span = to - from
+        if span <= 2.hours then "1m"
+        elsif span <= 12.hours then "5m"
+        elsif span <= 7.days then "1h"
+        else "6h"
+        end
+      end
+
+      # STEPS keys that make sense for the window, finest first.
+      def self.steps_for(from, to)
+        span = to - from
+        STEPS.select { |_key, step| (span / step).between?(2, MAX_POINTS) }.keys
+      end
+
+      # Time-bucketed series for charts, one point per `step` from `from` to `to`
+      # with empty buckets filled in: [{t, count, errors, client_errors, avg,
+      # p50, p95, p99}]. Durations in milliseconds; an empty bucket carries nil
+      # for them so a latency line breaks instead of dropping to zero.
+      def self.series(environment, record_type, from:, to:, group_hash: nil, step: nil, tenant: nil)
+        environment.with_telemetry { points(record_type, from: from, to: to, group_hash: group_hash, step: step, tenant: tenant) }
+      end
+
+      # `series` for a caller already inside environment.with_telemetry. A
+      # tenant narrows to one app_tenant, which rollups do not carry, so a
+      # tenant series reads raw rows at every step.
+      def self.points(record_type, from:, to:, group_hash: nil, step: nil, tenant: nil)
+        step = STEPS.fetch(step || default_step(from, to))
+        raw = step < 1.hour || tenant
+        points = raw ? raw_series(record_type, from, to, group_hash, step, tenant) : rollup_series(record_type, from, to, group_hash, step)
+        fill(points, from, to, step)
+      end
+
+      # Sums hourly rollups into `step`-wide buckets. Percentiles are the
+      # worst hour's, the same reading series always gave across groups.
+      def self.rollup_series(record_type, from, to, group_hash, step)
+        bucket = bucket_sql("bucket", step)
+        scope = Telemetry::Rollup.for_type(record_type).between(from, to)
+        scope = scope.where(group_hash: group_hash) if group_hash
+        scope.group(bucket).order(bucket)
+             .pluck(bucket, Arel.sql("SUM(count)"), Arel.sql("SUM(error_count)"), Arel.sql("SUM(client_error_count)"), Arel.sql("SUM(duration_sum)"), Arel.sql("MAX(p50)"), Arel.sql("MAX(p95)"), Arel.sql("MAX(p99)"))
+             .map { |b, c, e, ce, ds, p50, p95, p99| point(b, c, e, ce, ds, p50, p95, p99) }
+      end
+
+      # What a raw row of each record type is, and when it counts as an error
+      # (or a client error), mirroring RollupJob#error?. Both readings of the
+      # same rows must agree, which spec/models/telemetry/aggregations_spec.rb
+      # checks for every type.
+      RAW = {
+        "request" => [ -> { Telemetry::Execution.requests }, "status >= 500", "status BETWEEN 400 AND 499" ],
+        "job_attempt" => [ -> { Telemetry::Execution.jobs }, "outcome = 'failed'" ],
+        "scheduled_task" => [ -> { Telemetry::Execution.scheduled }, "outcome = 'failed'" ],
+        "command" => [ -> { Telemetry::Execution.commands }, "status != 0" ],
+        "channel_action" => [ -> { Telemetry::Execution.channels }, "outcome = 'failed'" ],
+        "query" => [ -> { Telemetry::Query.all } ],
+        "outgoing_request" => [ -> { Telemetry::OutgoingRequest.all }, "status_code >= 500 OR status_code IS NULL OR status_code = 0" ],
+        "cache_event" => [ -> { Telemetry::CacheEvent.all } ],
+        "mail" => [ -> { Telemetry::Mail.all }, "failed" ],
+        "visit" => [ -> { Telemetry::Visit.all }, "status = 'error'" ],
+        "span" => [ -> { Telemetry::Span.all }, "status = 'failed'" ],
+        "notification" => [ -> { Telemetry::Notification.all }, "failed" ],
+        "view_render" => [ -> { Telemetry::ViewRender.all } ],
+        "transaction" => [ -> { Telemetry::Transaction.all }, "outcome = 'rollback'" ],
+        "llm_call" => [ -> { Telemetry::LlmCall.models }, "status = 'failed'" ],
+        "llm_tool" => [ -> { Telemetry::LlmCall.tools }, "status = 'failed'" ]
+      }.freeze
+
+      # Sub-hour buckets straight from the raw rows, in one statement:
+      # counts and sums per bucket, and exact nearest-rank percentiles from a
+      # ROW_NUMBER over each bucket's durations. Rollups are hourly, so this
+      # is the only reading finer than an hour; it is also the freshest one,
+      # since the current hour's rollup is up to a minute behind.
+      def self.raw_series(record_type, from, to, group_hash, step, tenant = nil)
+        scope, error_sql, client_error_sql = RAW.fetch(record_type)
+        rows = scope.call.where(occurred_at: from..to)
+        rows = rows.where(group_hash: group_hash) if group_hash
+        rows = rows.where(app_tenant: tenant) if tenant
+        model = rows.model
+        inner = rows.select(bucket_sql("occurred_at", step).to_s + " AS b", "duration", flag_sql(error_sql) + " AS err", flag_sql(client_error_sql) + " AS cerr")
+        ranked = model.unscoped.from(inner, "r").select("b", "duration", "err", "cerr",
+          "ROW_NUMBER() OVER (PARTITION BY b ORDER BY duration) AS rn", "COUNT(*) OVER (PARTITION BY b) AS n")
+        model.unscoped.from(ranked, "w").group("b").order("b")
+          .pluck(Arel.sql("b"), Arel.sql("COUNT(*)"), Arel.sql("SUM(err)"), Arel.sql("SUM(cerr)"), Arel.sql("SUM(duration)"),
+                 rank_sql(50), rank_sql(95), rank_sql(99))
+          .map { |b, c, e, ce, ds, p50, p95, p99| point(b, c, e, ce, ds, p50, p95, p99) }
+      end
+
+      # The timestamp columns a series buckets on: raw rows by occurred_at,
+      # rollups and release health by their hourly bucket.
+      BUCKET_COLUMNS = %w[occurred_at bucket].freeze
+
+      # Epoch seconds of the start of the `step`-wide bucket holding `column`.
+      # `column` must be one of BUCKET_COLUMNS and `step` is a Duration, so the
+      # fragment can only ever hold a listed column name and an integer literal.
+      def self.bucket_sql(column, step)
+        raise ArgumentError, "unknown bucket column #{column.inspect}" unless BUCKET_COLUMNS.include?(column)
+        seconds = Integer(step.to_i)
+        Arel.sql("(strftime('%s', #{column}) / #{seconds}) * #{seconds}")
+      end
+
+      # One point per bucket from `from` to `to`, keeping the computed ones and
+      # zero-filling the rest, so a quiet minute is a gap of the right width.
+      # A series with its own point shape passes a block building its empty one.
+      def self.fill(points, from, to, step)
+        by_bucket = points.index_by { |p| p[:t] }
+        first = Time.at((from.to_i / step.to_i) * step.to_i).utc
+        last = Time.at((to.to_i / step.to_i) * step.to_i).utc
+        (first.to_i..last.to_i).step(step.to_i).map do |t|
+          at = Time.at(t).utc
+          by_bucket[at] || (block_given? ? yield(at) : empty_point(at))
         end
       end
 
+      def self.point(bucket, count, errors, client_errors, duration_sum, p50, p95, p99)
+        { t: Time.at(bucket).utc, count: count, errors: errors, client_errors: client_errors,
+         avg: count.zero? ? 0 : duration_sum / count / 1000.0, p50: p50 / 1000.0, p95: p95 / 1000.0, p99: p99 / 1000.0 }
+      end
+
+      def self.empty_point(t)
+        { t: t, count: 0, errors: 0, client_errors: 0, avg: nil, p50: nil, p95: nil, p99: nil }
+      end
+
+      def self.flag_sql(predicate)
+        predicate ? "CASE WHEN #{predicate} THEN 1 ELSE 0 END" : "0"
+      end
+
+      # The nearest-rank percentile: the duration whose rank is ceil(n * p/100).
+      RANK_SQL = { 50 => Arel.sql("MAX(CASE WHEN rn = (n * 50 + 99) / 100 THEN duration END)"),
+                  95 => Arel.sql("MAX(CASE WHEN rn = (n * 95 + 99) / 100 THEN duration END)"),
+                  99 => Arel.sql("MAX(CASE WHEN rn = (n * 99 + 99) / 100 THEN duration END)") }.freeze
+
+      def self.rank_sql(percent)
+        RANK_SQL.fetch(percent)
+      end
+
+      private_class_method :rollup_series, :raw_series, :point, :empty_point, :flag_sql, :rank_sql
+
       # Per-group table rows for a record type in the window: count/error totals,
       # merged percentiles (via Telemetry::Rollup.summarize), and a sparkline.
       def self.grouped(environment, record_type, from:, to:, limit: 100, order: nil, dir: nil)
@@ -34,6 +170,15 @@ module Railwatch
         cached(key) { grouped_uncached(environment, record_type, from: from, to: to, limit: limit, order: order, sign: sign) }
       end
 
+      # The minute cache exists because the platform's rollups only change
+      # when RollupJob writes. Embedded, every batch updates them and one
+      # person is looking, so the cache would only make the page a minute
+      # stale for nothing.
+      def self.cached(key, &block)
+        return yield if Railwatch.config.local?
+        Rails.cache.fetch(key, expires_in: 1.minute, &block)
+      end
+
       def self.grouped_uncached(environment, record_type, from:, to:, limit:, order:, sign:)
         environment.with_telemetry do
           rows = Telemetry::Rollup.for_type(record_type).between(from, to).to_a
@@ -60,16 +205,6 @@ module Railwatch
         end
       end
 
-      # The minute cache exists because the platform's rollups only change
-      # when RollupJob writes. Embedded, every batch updates them and one
-      # person is looking, so the cache would only make the page a minute
-      # stale for nothing.
-      def self.cached(key, &block)
-        return yield if Railwatch.config.local?
-
-        Rails.cache.fetch(key, expires_in: 1.minute, &block)
-      end
-
       # The digest-free half of Rollup.summarize: everything the sort keys
       # that are not percentiles need.
       def self.cheap_summary(rows)

data/app/models/railwatch/telemetry/exception.rb

@@ -9,6 +9,13 @@ module Railwatch
 
       scope :unhandled, -> { where(handled: false) }
 
+      def as_row
+        { id: id, class_name: class_name, message: message.first(500), handled: handled, severity: severity, source: source,
+          file: file, line: line, occurred_at: occurred_at, deploy: deploy, execution_id: execution_id,
+          execution_source: execution_source, execution_preview: execution_preview, user_ref: user_ref, tenant: app_tenant,
+          group_hash: group_hash }
+      end
+
       def timeline_label
         "#{class_name}: #{message.to_s.first(100)}"
       end

data/app/models/railwatch/telemetry/execution.rb

@@ -63,6 +63,24 @@ module Railwatch
         duration / 1000.0
       end
 
+      def queue_latency_ms
+        queue_latency && (queue_latency / 1000.0).round(1)
+      end
+
+      # The row every list surface shows for an execution: what the kind has
+      # in common, then what it adds.
+      def as_row
+        row = { execution_id: execution_id, kind: kind, name: name, status: status, outcome: outcome, duration: duration_ms.round(2),
+                occurred_at: occurred_at, deploy: deploy, server: server, user_ref: user_ref, tenant: app_tenant,
+                exception_preview: exception_preview }
+        case kind
+        when "request" then row.merge(method: self[:method], inertia_component: inertia_component, queries: counters["queries"])
+        when "job_attempt" then row.merge(queue: queue, attempt: attempt, job_id: job_id, queue_latency: queue_latency_ms)
+        when "scheduled_task" then row.merge(task_key: task_key)
+        else row
+        end
+      end
+
       def children_count
         counters.values.sum
       end

data/app/models/railwatch/telemetry/log.rb

@@ -77,6 +77,12 @@ module Railwatch
         def quote(term) = %("#{term.gsub('"', '""')}")
       end
 
+      def as_row
+        { id: id, level: level, message: message.first(2000), tags: tags, occurred_at: occurred_at, deploy: deploy,
+          execution_id: execution_id, execution_source: execution_source, execution_preview: execution_preview,
+          source: source, tenant: app_tenant, user_ref: user_ref, context: context }
+      end
+
       def timeline_label
         message.to_s.first(120)
       end

data/app/models/railwatch/telemetry/query.rb

@@ -42,6 +42,13 @@ module Railwatch
         text
       end
 
+      def as_row
+        { id: id, group_hash: group_hash, sql: sql.first(2_000), name: name, duration: duration_ms.round(3), occurred_at: occurred_at,
+          deploy: deploy, execution_id: execution_id, execution_preview: execution_preview, source: source,
+          connection: connection, role: role, adapter: adapter, row_count: row_count, tenant: app_tenant, user_ref: user_ref,
+          explain: explain.present? }
+      end
+
       def timeline_label
         sql.to_s.first(120)
       end

data/app/models/railwatch/telemetry/release_health.rb

@@ -36,13 +36,19 @@ module Railwatch
         }
       end
 
-      # Hourly points for the stacked sessions-by-status chart.
-      def self.series(deploy, from, to)
-        between(from, to).for_release(deploy).group(:bucket).order(:bucket)
-          .pluck(:bucket, Arel.sql("SUM(sessions)"), Arel.sql("SUM(sessions_errored)"), Arel.sql("SUM(sessions_crashed)"))
-          .map { |bucket, sessions, errored, crashed|
-            { t: bucket, sessions: sessions, ok: sessions - errored - crashed, errored: errored, crashed: crashed }
+      # Points for the stacked sessions-by-status chart, one per `step` across
+      # the window. The table is hourly and a session is counted once per hour
+      # it was seen, so a step under an hour is drawn at an hour: raw session
+      # rows would count a live session on every flush.
+      def self.series(deploy, from, to, step: nil)
+        step = [ Telemetry::Aggregations::STEPS.fetch(step || Telemetry::Aggregations.default_step(from, to)), 1.hour ].max
+        bucket = Telemetry::Aggregations.bucket_sql("bucket", step)
+        points = between(from, to).for_release(deploy).group(bucket).order(bucket)
+          .pluck(bucket, Arel.sql("SUM(sessions)"), Arel.sql("SUM(sessions_errored)"), Arel.sql("SUM(sessions_crashed)"))
+          .map { |b, sessions, errored, crashed|
+            { t: Time.at(b).utc, sessions: sessions, ok: sessions - errored - crashed, errored: errored, crashed: crashed }
           }
+        Telemetry::Aggregations.fill(points, from, to, step) { |t| { t: t, sessions: 0, ok: 0, errored: 0, crashed: 0 } }
       end
 
       # Each release's share of the window's sessions, as a percentage -- how

data/app/models/railwatch/telemetry/rollup.rb

@@ -54,7 +54,7 @@ module Railwatch
       # Aggregate a relation of rollups into one summary with merged percentiles.
       def self.summarize(relation)
         rows = relation.to_a
-        return { count: 0, errors: 0, client_errors: 0, avg: 0, p50: 0, p95: 0, p99: 0, max: 0 } if rows.empty?
+        return { count: 0, errors: 0, client_errors: 0, avg: 0, p50: 0, p95: 0, p99: 0, max: 0, extra: {} } if rows.empty?
         # merge! pushes the row's centroids into one accumulating digest.
         # `+` built a brand-new digest from both operands' centroids on every
         # row, so merging N rows re-pushed every earlier centroid N times:
@@ -70,9 +70,24 @@ module Railwatch
           p50: merged.percentile(0.5).to_i,
           p95: merged.percentile(0.95).to_i,
           p99: merged.percentile(0.99).to_i,
-          max: rows.map(&:duration_max).max
+          max: rows.map(&:duration_max).max,
+          # Everything a type puts in `extra` -- llm_call's cost_nanos and
+          # token counts, cache_event's hits and misses -- merged the way
+          # absorb! merges it: numbers add up, anything else is last-wins.
+          # Spend is a sum over a window rather than a percentile of
+          # durations, so a rule about money has nowhere else to read from.
+          extra: merge_extras(rows)
         }
       end
+
+      def self.merge_extras(rows)
+        rows.each_with_object({}) do |row, out|
+          row.extra.each do |key, value|
+            existing = out[key]
+            out[key] = existing.is_a?(Numeric) && value.is_a?(Numeric) ? existing + value : value
+          end
+        end
+      end
     end
   end
 end

data/app/models/railwatch/telemetry/tenant.rb

@@ -18,7 +18,6 @@ module Railwatch
       # aggregate), so only the busiest tenants get one; the rest report 0.
       P95_TENANTS = 50
       SPARKLINE_BUCKETS = 12
-      SERIES_BUCKETS = 48
       SORTS = { "requests" => :requests, "errors" => :errors, "p95" => :p95, "users" => :users }.freeze
 
       ERRORS_SQL = Arel.sql("SUM(CASE WHEN status >= 500 THEN 1 ELSE 0 END)")
@@ -71,21 +70,12 @@ module Railwatch
         }
       end
 
-      # Request volume and duration bucketed over the window, SeriesPoint-shaped
-      # so the same charts render it. Percentiles are approximations: these
-      # buckets come from raw rows, without the per-bucket t-digest
-      # Telemetry::Rollup keeps, so p50 is the bucket's average and p95/p99 its
-      # max.
-      def self.series(tenant, from, to)
-        width = bucket_width(from, to, SERIES_BUCKETS)
-        bucket = bucket_sql(from, width)
-        Telemetry::Execution.requests.between(from, to).where(app_tenant: tenant).group(bucket)
-          .pluck(bucket, COUNT_SQL, ERRORS_SQL, CLIENT_ERRORS_SQL, Arel.sql("SUM(duration)"), Arel.sql("MAX(duration)"))
-          .map { |index, count, errors, client_errors, sum, max|
-            avg = ms(sum / count)
-            { t: bucket_at(from, width, index, SERIES_BUCKETS).iso8601, count: count, errors: errors, client_errors: client_errors,
-              avg: avg, p50: avg, p95: ms(max), p99: ms(max) }
-          }.sort_by { |point| point[:t] }
+      # Request volume and duration bucketed over the window at `step`,
+      # SeriesPoint-shaped so the same charts render it. Rollups carry no tenant,
+      # so every step reads raw rows (Telemetry::Aggregations.points does that
+      # for a tenant), with exact percentiles per bucket.
+      def self.series(tenant, from, to, step: nil)
+        Telemetry::Aggregations.points("request", from: from, to: to, step: step, tenant: tenant)
       end
 
       def self.routes(tenant, from, to, limit: 20)
@@ -109,14 +99,9 @@ module Railwatch
         end
       end
 
-      # Same field shape as RequestsController#execution_row, for the tenant
-      # page's "Recent requests" table.
+      # The tenant page's "Recent requests" table.
       def self.recent_requests(tenant, from, to, limit: 50)
-        Telemetry::Execution.requests.between(from, to).where(app_tenant: tenant).recent.limit(limit).map do |r|
-          { execution_id: r.execution_id, name: r.name, status: r.status, duration: r.duration_ms.round(2), occurred_at: r.occurred_at,
-            user_ref: r.user_ref, tenant: r.app_tenant, exception_preview: r.exception_preview, inertia_component: r.inertia_component,
-            queries: r.counters["queries"], deploy: r.deploy }
-        end
+        Telemetry::Execution.requests.between(from, to).where(app_tenant: tenant).recent.limit(limit).map(&:as_row)
       end
 
       # -- Aggregation steps -----------------------------------------------------

data/app/models/railwatch/telemetry_record.rb

@@ -2,12 +2,14 @@
 
 module Railwatch
   # Base class for everything the engine stores about the host application.
-  # The hosted platform keeps one SQLite file per monitored environment through
-  # activerecord-tenanted; an embedded install monitors exactly one application,
-  # so this is a plain second database (`railwatch_telemetry` in the host's
-  # database.yml), separate from the host's own primary and from the engine's
-  # meta tables. Every telemetry query still runs inside
-  # Environment#with_telemetry so the code path matches the platform's.
+  # An embedded install monitors exactly one application, so this is a plain
+  # second database (`railwatch_telemetry` in the host's database.yml),
+  # separate from the host's own primary and from the engine's meta tables.
+  # The hosted platform keeps one SQLite file per monitored environment
+  # instead: it calls `Railwatch::TelemetryRecord.tenanted("telemetry")`
+  # (activerecord-tenanted) from an initializer, which replaces the
+  # connection below with a per-tenant one, and every telemetry query runs
+  # inside Environment#with_telemetry so the code path is the same in both.
   class TelemetryRecord < ActiveRecord::Base
     self.abstract_class = true
     begin

data/app/models/railwatch/threshold.rb

@@ -7,8 +7,22 @@ module Railwatch
     self.table_name = "railwatch_thresholds"
     MAX_LIMIT = 1_000_000_000
     MAX_WINDOW_MINUTES = 1_440
-    TARGET_KINDS = %w[requests jobs commands queries scheduled_tasks outgoing_requests].freeze
-    METRICS = %w[p95 max avg error_rate failure_rate missed].freeze
+    TARGET_KINDS = %w[requests jobs commands queries scheduled_tasks outgoing_requests
+                      llm_calls llm_tools].freeze
+    METRICS = %w[p95 max avg error_rate failure_rate missed spend tokens truncation_rate].freeze
+
+    # Duration and rate metrics read the same on any timed record, so they are
+    # allowed everywhere. Money and tokens only mean something where a record
+    # carries them: a spend rule on tool calls could never fire, and a rule
+    # that cannot fire is worse than no rule -- it reads as coverage.
+    SHARED_METRICS = %w[p95 max avg error_rate failure_rate missed].freeze
+    LLM_METRICS = %w[spend tokens truncation_rate].freeze
+
+    # Money leads with its symbol; milliseconds and percent follow the number.
+    # The old code assumed two units and branched on the metric name ending in
+    # "rate", which cannot express a dollar sign in front.
+    UNITS = { "spend" => "$", "tokens" => "", "truncation_rate" => "%",
+             "error_rate" => "%", "failure_rate" => "%" }.freeze
 
     def environment = Environment.current
 
@@ -20,10 +34,33 @@ module Railwatch
     validates :target, presence: true, length: { maximum: 256 }
     validates :limit, numericality: { less_than_or_equal_to: 100 }, if: -> { metric&.end_with?("rate") }
     validates :target, uniqueness: { scope: %i[environment_id target_kind metric] }
+    validate :metric_applies_to_target_kind
+
+    def self.metrics_for(target_kind)
+      target_kind == "llm_calls" ? SHARED_METRICS + LLM_METRICS : SHARED_METRICS
+    end
+
+    def unit = UNITS.fetch(metric, "ms")
+
+    # "$4.50", "2000ms", "5%", "150000" -- the symbol's position is part of
+    # the unit, not something a caller should have to know.
+    def format_value(value)
+      return "$#{value.is_a?(Float) ? format('%.2f', value) : value}" if unit == "$"
+
+      "#{value}#{unit}"
+    end
 
     def description
-      unit = metric.end_with?("rate") ? "%" : "ms"
-      "#{target_kind} #{target == '*' ? 'all' : target}: #{metric} over #{limit}#{unit} in #{window_minutes}m"
+      "#{target_kind} #{target == '*' ? 'all' : target}: #{metric} over #{format_value(limit)} in #{window_minutes}m"
+    end
+
+    private
+
+    def metric_applies_to_target_kind
+      return if metric.blank? || target_kind.blank?
+      return if Threshold.metrics_for(target_kind).include?(metric)
+
+      errors.add(:metric, "is not available for #{target_kind}")
     end
   end
 end

data/app/models/railwatch/window.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Railwatch
+  # The time range a telemetry page, API call, or MCP tool looks at: one of the
+  # fixed presets ("1h" .. "30d") ending now, or a custom [from, to] pair. The
+  # dashboard, the JSON API, and MCP all resolve their ?window= / "window"
+  # argument through here so the presets and the default agree everywhere.
+  class Window
+    PRESETS = { "1h" => 1.hour, "6h" => 6.hours, "24h" => 24.hours, "7d" => 7.days, "30d" => 30.days }.freeze
+    DEFAULT = "24h"
+    MAX_CUSTOM_RANGE = 90.days
+
+    attr_reader :key, :from, :to
+
+    # A preset key, or anything else for the default.
+    def self.preset(key)
+      key = PRESETS.key?(key.to_s) ? key.to_s : DEFAULT
+      to = Time.current
+      new(key, to - PRESETS[key], to)
+    end
+
+    # A custom range from ?from=&to= when both parse, are in order, and span
+    # no more than MAX_CUSTOM_RANGE; otherwise the preset (or default).
+    def self.parse(window: nil, from: nil, to: nil)
+      custom(from, to) || preset(window)
+    end
+
+    def self.custom(from, to)
+      from = parse_time(from) or return
+      to = to.blank? ? Time.current : parse_time(to) or return
+      new("custom", from, to) if to > from && (to - from) <= MAX_CUSTOM_RANGE
+    end
+
+    def self.parse_time(value)
+      return if value.blank?
+      Time.zone.parse(value.to_s)
+    rescue ArgumentError, TypeError
+      nil
+    end
+    private_class_method :parse_time
+
+    def initialize(key, from, to)
+      @key = key
+      @from = from
+      @to = to
+    end
+
+    def custom? = key == "custom"
+    def range = [ from, to ]
+    def span = to - from
+
+    # The same length immediately before this one, for "vs. previous period".
+    def previous
+      self.class.new(key, from - span, from)
+    end
+
+    def to_h
+      { from: from.iso8601(6), to: to.iso8601(6) }
+    end
+
+    # Query params that reproduce this window on another page.
+    def to_params
+      custom? ? to_h : { window: key }
+    end
+  end
+end