sentiero 1.0.0.alpha1

data/lib/sentiero/analytics/problem_detail.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require "date"
+
+module Sentiero
+  module Analytics
+    # Aggregations for the problem detail page: facet distributions over the
+    # already-fetched occurrences/session summaries, and the occurrence trend.
+    # The trend's rolling counts query the store (count_occurrences); the facets
+    # and the sparkline buckets are computed purely from the passed-in rows.
+    class ProblemDetail
+      # Rows shown per facet group on the problem detail page.
+      FACET_LIMIT = 8
+
+      # Day buckets in the problem-detail occurrence sparkline.
+      TREND_DAYS = 30
+
+      def initialize(store)
+        @store = store
+      end
+
+      def facets(occurrences, session_summaries)
+        paths = Hash.new(0)
+        environments = Hash.new(0)
+        releases = {}
+        browsers = Hash.new(0)
+
+        occurrences.each do |occ|
+          ctx = occ["context"]
+          next unless ctx.is_a?(Hash)
+          tally_facet(paths, ctx.dig("request", "path"))
+          tally_facet(environments, ctx["environment"])
+          tally_release(releases, ctx["release"], occ["timestamp"])
+        end
+
+        session_summaries.each { |s| tally_facet(browsers, s[:browser]) }
+
+        {
+          paths: top_facet(paths),
+          environments: top_facet(environments),
+          releases: releases.sort_by { |_release, info| -info[:count] }.first(FACET_LIMIT),
+          browsers: top_facet(browsers),
+          sample_size: occurrences.size
+        }
+      end
+
+      # The 24h/7d/30d header counts are exact (count_occurrences after:); the
+      # sparkline buckets the already-fetched occurrences by UTC day, labeled
+      # with its sample size.
+      def trend(problem_id, occurrences)
+        now = Time.now.to_f
+        per_day = Hash.new(0)
+        occurrences.each do |occ|
+          ts = occ["timestamp"]&.to_f
+          next unless ts && ts > 0
+          per_day[Time.at(ts).utc.to_date.to_s] += 1
+        end
+
+        end_date = Time.now.utc.to_date
+        series = ((end_date - (TREND_DAYS - 1))..end_date).map do |date|
+          {date: date.to_s, count: per_day[date.to_s]}
+        end
+
+        {
+          series: series,
+          sample_size: occurrences.size,
+          last_24h: occurrence_count_after(problem_id, now - 86_400),
+          last_7d: occurrence_count_after(problem_id, now - 7 * 86_400),
+          last_30d: occurrence_count_after(problem_id, now - TREND_DAYS * 86_400)
+        }
+      end
+
+      private
+
+      def occurrence_count_after(problem_id, after)
+        @store.count_occurrences(problem_id, after: after)
+      end
+
+      def tally_facet(counts, value)
+        counts[value] += 1 if value.is_a?(String) && !value.empty?
+      end
+
+      def tally_release(releases, release, timestamp)
+        return unless release.is_a?(String) && !release.empty?
+
+        info = releases[release] ||= {count: 0, first_seen: nil}
+        info[:count] += 1
+        ts = timestamp&.to_f
+        info[:first_seen] = [info[:first_seen], ts].compact.min if ts && ts > 0
+      end
+
+      def top_facet(counts)
+        counts.sort_by { |_value, count| -count }.first(FACET_LIMIT)
+      end
+    end
+  end
+end

data/lib/sentiero/analytics/scroll_depth_analyzer.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative "analyzer"
+require_relative "collectors/scroll_collector"
+
+module Sentiero
+  module Analytics
+    # Aggregates per-URL scroll depth across sessions (avg depth, fold lines,
+    # distribution). Depth math lives in ScrollCollector; this drives it per URL.
+    class ScrollDepthAnalyzer < Analyzer
+      MAX_URLS = 200
+
+      def analyze(limit: nil, since: nil, until_time: nil)
+        scroll = ScrollCollector.new(max_urls: MAX_URLS)
+
+        _scanned, hit_cap = scan_sessions(limit: limit, since: since, until_time: until_time) do |_summary, _window_id, events|
+          each_page_segment(events) do |url, segment, _anchor|
+            scroll.observe(url, segment) if url
+          end
+          scroll.flush_window
+        end
+
+        {
+          pages: scroll.pages,
+          was_truncated: scroll.capped || hit_cap
+        }
+      end
+    end
+  end
+end

data/lib/sentiero/analytics/segmenter.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require_relative "analyzer"
+require_relative "../user_agent"
+
+module Sentiero
+  module Analytics
+    # Filters sessions on read by browser/device/URL/metadata/has-errors/duration
+    # (AND-combined), scanning up to the store's limits.analytics_max_scan_sessions.
+    class Segmenter < Analyzer
+      def initialize(
+        store = Sentiero.store,
+        browser: nil,
+        device: nil,
+        url_pattern: nil,
+        metadata_key: nil,
+        metadata_value: nil,
+        metadata_match: "exact",
+        has_errors: false,
+        min_duration_ms: nil,
+        max_duration_ms: nil,
+        since: nil,
+        until_time: nil
+      )
+        super(store)
+        @browser = presence(browser)
+        @device = presence(device)
+        @url_pattern = presence(url_pattern)
+        @metadata_key = presence(metadata_key)
+        @metadata_value = presence(metadata_value)
+        @metadata_match = (metadata_match == "contains") ? "contains" : "exact"
+        @has_errors = has_errors
+        @min_duration_ms = min_duration_ms
+        @max_duration_ms = max_duration_ms
+        @since = since
+        @until_time = until_time
+      end
+
+      def matching(limit: 20, offset: 0)
+        scan_cap = store.limits.analytics_max_scan_sessions
+
+        scanned = store.list_sessions(limit: scan_cap, offset: 0, since: @since, until_time: @until_time)
+        matches = scanned.select { |summary| match?(summary) }
+
+        page = matches.slice(offset, limit + 1) || []
+        has_next = page.size > limit
+
+        {
+          sessions: page.first(limit),
+          has_next: has_next,
+          was_truncated: scanned.size >= scan_cap
+        }
+      end
+
+      private
+
+      def presence(value)
+        return nil unless value.is_a?(String)
+        stripped = value.strip
+        stripped.empty? ? nil : stripped
+      end
+
+      def match?(summary)
+        metadata = summary[:metadata] || {}
+
+        browser_match?(metadata) &&
+          device_match?(metadata) &&
+          url_match?(metadata) &&
+          metadata_match?(metadata) &&
+          has_errors_match?(metadata) &&
+          duration_match?(summary)
+      end
+
+      def browser_match?(metadata)
+        return true unless @browser
+
+        UserAgent.browser(metadata["userAgent"]) == @browser
+      end
+
+      def device_match?(metadata)
+        return true unless @device
+
+        UserAgent.device(metadata["userAgent"]) == @device
+      end
+
+      def url_match?(metadata)
+        return true unless @url_pattern
+
+        url = metadata["url"]
+        return false unless url.is_a?(String)
+
+        glob?(@url_pattern) ? glob_match?(url, @url_pattern) : url.downcase.include?(@url_pattern.downcase)
+      end
+
+      def glob?(pattern)
+        pattern.include?("*") || pattern.include?("?")
+      end
+
+      def glob_match?(url, pattern)
+        File.fnmatch(pattern, url, File::FNM_CASEFOLD)
+      end
+
+      def metadata_match?(metadata)
+        return true unless @metadata_key
+        return false unless metadata.key?(@metadata_key)
+        return true unless @metadata_value
+
+        value = metadata[@metadata_key].to_s
+        if @metadata_match == "contains"
+          value.downcase.include?(@metadata_value.downcase)
+        else
+          value == @metadata_value
+        end
+      end
+
+      def has_errors_match?(metadata)
+        return true unless @has_errors
+
+        metadata["has_errors"] == true
+      end
+
+      def duration_match?(summary)
+        return true unless @min_duration_ms || @max_duration_ms
+
+        duration = duration_ms(summary)
+        return false if duration.nil?
+        return false if @min_duration_ms && duration < @min_duration_ms
+        return false if @max_duration_ms && duration > @max_duration_ms
+        true
+      end
+    end
+  end
+end

data/lib/sentiero/analytics/server_event_metrics.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+require "date"
+
+module Sentiero
+  module Analytics
+    # Per-day aggregations over an already-fetched list of custom events for the
+    # events-index strips (level mix, numeric payload metrics). Operates purely on
+    # the passed-in rows — it never re-reads the store, so the dashboard fetches
+    # once and aggregates the full pre-pagination list here.
+    class ServerEventMetrics
+      # Most-recent day rows rendered in the events-index level-mix strip.
+      LEVEL_MIX_MAX_DAYS = 30
+      SERVER_EVENT_LEVELS = %w[debug info warn error].freeze
+
+      # Cap on distinct payload keys offered in the metric_key dropdown.
+      MAX_METRIC_KEYS = 50
+
+      # Adapts BrowserEventDiscovery rows (symbol-keyed, rrweb epoch-MILLISECOND
+      # timestamps) to the string-keyed, epoch-seconds shape these helpers expect,
+      # so the browser tab can reuse them.
+      def self.adapt_browser_rows(rows)
+        rows.map do |row|
+          {
+            "name" => row[:name],
+            "payload" => row[:payload],
+            "timestamp" => row[:timestamp] && (row[:timestamp].to_f / 1000.0)
+          }
+        end
+      end
+
+      def initialize(events)
+        @events = events
+      end
+
+      # Per-UTC-day level tallies. Returns [[date, {level => count}], ...]
+      # ascending, capped to the most recent LEVEL_MIX_MAX_DAYS days with data.
+      def level_mix_by_day
+        days = Hash.new { |hash, key| hash[key] = Hash.new(0) }
+        @events.each do |event|
+          ts = event["timestamp"]&.to_f
+          next unless ts && ts > 0
+          level = event["level"]
+          level = "info" unless SERVER_EVENT_LEVELS.include?(level)
+          days[Time.at(ts).utc.to_date.to_s][level] += 1
+        end
+        days.sort_by { |date, _counts| date }.last(LEVEL_MIX_MAX_DAYS)
+      end
+
+      # Payload metrics are offered only when the rows share a single event name,
+      # computed over those rows. `requested_key` is the user-selected metric key,
+      # honored only if it names a numeric payload key.
+      def payload_metric_locals(requested_key)
+        single_name = single_event_name
+        metric_keys = single_name ? numeric_payload_keys : []
+        metric_key = metric_keys.include?(requested_key) ? requested_key : nil
+
+        {
+          single_name: single_name,
+          metric_keys: metric_keys,
+          metric_key: metric_key,
+          metric_days: metric_key ? payload_metrics_by_day(metric_key) : []
+        }
+      end
+
+      private
+
+      # The shared event name when every row carries the same one — the
+      # precondition for offering payload metrics.
+      def single_event_name
+        return nil if @events.empty?
+        name = @events.first["name"]
+        return nil unless name.is_a?(String) && !name.empty?
+        (@events.all? { |event| event["name"] == name }) ? name : nil
+      end
+
+      # Payload keys observed with at least one Numeric value across the rows,
+      # sorted; distinct keys capped at MAX_METRIC_KEYS.
+      def numeric_payload_keys
+        keys = {}
+        @events.each do |event|
+          payload = event["payload"]
+          next unless payload.is_a?(Hash)
+          payload.each do |key, value|
+            next unless value.is_a?(Numeric)
+            next if !keys.key?(key) && keys.size >= MAX_METRIC_KEYS
+            keys[key] = true
+          end
+        end
+        keys.keys.sort
+      end
+
+      # Per-UTC-day count/sum/min/max of one payload key (mirrors level_mix_by_day).
+      # Non-numeric values are skipped and tallied separately. Returns
+      # [[date, {count:, sum:, min:, max:, non_numeric:}], ...] ascending.
+      def payload_metrics_by_day(key)
+        days = Hash.new { |hash, date| hash[date] = {count: 0, sum: 0.0, min: nil, max: nil, non_numeric: 0} }
+        @events.each do |event|
+          ts = event["timestamp"]&.to_f
+          next unless ts && ts > 0
+          payload = event["payload"]
+          next unless payload.is_a?(Hash) && payload.key?(key)
+
+          day = days[Time.at(ts).utc.to_date.to_s]
+          value = payload[key]
+          unless value.is_a?(Numeric)
+            day[:non_numeric] += 1
+            next
+          end
+
+          day[:count] += 1
+          day[:sum] += value
+          day[:min] = day[:min] ? [day[:min], value].min : value
+          day[:max] = day[:max] ? [day[:max], value].max : value
+        end
+        days.sort_by { |date, _metrics| date }.last(LEVEL_MIX_MAX_DAYS)
+      end
+    end
+  end
+end

data/lib/sentiero/analytics/stats.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Sentiero
+  module Analytics
+    module Stats
+      # Nearest-rank percentile; `sorted` must be pre-sorted and non-empty.
+      def percentile(sorted, pct)
+        rank = (pct / 100.0 * sorted.size).ceil
+        sorted[rank.clamp(1, sorted.size) - 1]
+      end
+
+      def mean(values)
+        values.sum.to_f / values.size
+      end
+
+      # Milliseconds from `anchor` to `timestamp`, floored at 0; 0 if either is nil.
+      def offset_ms(anchor, timestamp)
+        return 0 unless anchor && timestamp
+
+        [timestamp - anchor, 0].max.round
+      end
+
+      # Top `limit` [key, count] pairs, highest count first, ties broken by key
+      # so the ordering is deterministic.
+      def top_counts(counts, limit:)
+        counts.sort_by { |key, count| [-count, key] }.first(limit)
+      end
+    end
+  end
+end

data/lib/sentiero/analytics/stats_aggregator/result_builder.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+module Sentiero
+  module Analytics
+    class StatsAggregator
+      # Pure presentation layer: turns a finished Accumulator into the
+      # aggregate's public result hash. Needs the store only for the two
+      # problem lookups (open_problems count and the top_problems list).
+      class ResultBuilder
+        include Events
+        include Stats
+
+        def initialize(store)
+          @store = store
+        end
+
+        def build(acc, sessions_scanned, scan_cap, server_overlay_truncated)
+          open_problems = store.list_problems(project: nil, limit: 10_000, status: "open")
+          groups, series_bucket = bucket_groups(acc)
+          series = groups.map { |label, dates| series_entry(acc, label, dates) }
+          custom_event_tags = top_tags(acc.custom_tags.tags)
+
+          {
+            total_sessions: sessions_scanned,
+            total_events: acc.total_events,
+            sessions_scanned: sessions_scanned,
+            avg_duration_ms: average(acc.durations),
+            open_problems: open_problems.size,
+            top_problems: top_problems(open_problems, acc.since),
+            sessions_with_errors: acc.sessions_with_errors,
+            event_type_breakdown: event_type_breakdown(acc.event_types),
+            browser_distribution: sort_by_count(acc.browsers),
+            device_distribution: sort_by_count(acc.devices),
+            top_entry_pages: top_entry_pages(acc),
+            top_referrers: top_list(acc.referrers, :referrer, StatsAggregator::TOP_LIST_LIMIT),
+            session_duration_buckets: acc.duration_buckets,
+            custom_event_tags: custom_event_tags,
+            custom_event_tag_series: tag_series(acc, custom_event_tags, groups),
+            browser_event_tags: acc.browser_tags,
+            navigation: {
+              internal: top_list(acc.nav_internal, :url, StatsAggregator::TOP_LIST_LIMIT),
+              external: top_list(acc.nav_external, :url, StatsAggregator::TOP_LIST_LIMIT),
+              top_texts: top_list(acc.nav_texts, :text, StatsAggregator::TOP_LIST_LIMIT)
+            },
+            metadata_distributions: metadata_distributions(acc),
+            events_per_day_series: series,
+            series_bucket: series_bucket,
+            # True when a cap was hit: server_error_count values are then a lower bound.
+            server_overlay_truncated: server_overlay_truncated,
+            # Effective bounds (unbounded "until" resolves to now) for period-over-period.
+            window_since: acc.since,
+            window_until: acc.until_time || Time.now.to_f,
+            was_truncated: sessions_scanned >= scan_cap
+          }
+        end
+
+        private
+
+        attr_reader :store
+
+        def top_problems(open_problems, since)
+          open_problems
+            .sort_by { |problem| -problem[:count].to_i }
+            .first(StatsAggregator::TOP_PROBLEMS_LIMIT)
+            .map do |problem|
+              {
+                id: problem[:id],
+                exception_class: problem[:exception_class],
+                message: problem[:message],
+                count: problem[:count],
+                first_seen: problem[:first_seen],
+                new: !!(problem[:first_seen] && problem[:first_seen] >= since)
+              }
+            end
+        end
+
+        def event_type_breakdown(types)
+          {
+            incremental: types[INCREMENTAL],
+            meta: types[META],
+            custom: types[CUSTOM]
+          }
+        end
+
+        def average(values)
+          return nil if values.empty?
+          values.sum / values.size.to_f
+        end
+
+        def sort_by_count(counts)
+          top_counts(counts, limit: counts.size).to_h
+        end
+
+        def top_list(counts, key, limit)
+          top_counts(counts, limit: limit).map { |value, count| {key => value, :count => count} }
+        end
+
+        def top_tags(counts)
+          top_counts(counts, limit: StatsAggregator::TOP_TAGS_LIMIT).to_h
+        end
+
+        def metadata_distributions(acc)
+          acc.metadata_keys
+            .sort_by { |_key, count| -count }
+            .first(StatsAggregator::TOP_LIST_LIMIT)
+            .map do |key, count|
+              {key: key, count: count, values: top_list(acc.metadata_values[key] || {}, :value, 5)}
+            end
+        end
+
+        def top_entry_pages(acc)
+          top_list(acc.entry_pages, :url, StatsAggregator::TOP_LIST_LIMIT).map do |row|
+            row.merge(error_count: acc.entry_page_errors[row[:url]])
+          end
+        end
+
+        # One bucket per UTC day, or per ISO week past WEEK_BUCKET_THRESHOLD_DAYS.
+        # Main and per-tag series share these groups so they align.
+        def bucket_groups(acc)
+          start_date = Time.at(acc.since).utc.to_date
+          end_date = (acc.until_time ? Time.at(acc.until_time) : Time.now).utc.to_date
+          end_date = start_date if end_date < start_date
+          days = (start_date..end_date).to_a
+
+          if days.size > StatsAggregator::WEEK_BUCKET_THRESHOLD_DAYS
+            [days.group_by { |date| date.strftime("%G-W%V") }.to_a, "week"]
+          else
+            [days.map { |date| [date.to_s, [date]] }, "day"]
+          end
+        end
+
+        def tag_series(acc, custom_event_tags, groups)
+          custom_event_tags.keys.filter_map do |tag|
+            per_day = acc.per_day_tags[tag]
+            next unless per_day
+
+            [tag, groups.map { |label, dates| {date: label, count: dates.sum { |date| per_day[date.to_s] }} }]
+          end.to_h
+        end
+
+        def series_entry(acc, label, dates)
+          {
+            date: label,
+            event_count: dates.sum { |date| acc.per_day_events[date.to_s] },
+            session_count: dates.sum { |date| acc.per_day_sessions[date.to_s] },
+            error_count: dates.sum { |date| acc.per_day_errors[date.to_s] },
+            server_error_count: dates.sum { |date| acc.per_day_server_errors[date.to_s] }
+          }
+        end
+      end
+    end
+  end
+end