sentiero 1.0.0.alpha1

data/lib/sentiero/analytics/analyzer.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require_relative "events"
+require_relative "stats"
+require_relative "bounded"
+require_relative "entry_attribution"
+
+module Sentiero
+  # Compute-on-read analytics: query the store and aggregate at request time.
+  module Analytics
+    class Analyzer
+      include Events
+      include Stats
+      include Bounded
+      include EntryAttribution
+
+      attr_reader :store
+
+      def initialize(store = Sentiero.store)
+        @store = store
+      end
+
+      private
+
+      # The standard bounded session scan: yields each window's
+      # [summary, window_id, events] up to the scan cap, and returns
+      # [sessions_scanned, hit_cap]. Counts DISTINCT sessions (not windows), so
+      # `hit_cap` is correct even when a session spans several windows. Callers
+      # build was_truncated as `collector.capped || hit_cap`.
+      def scan_sessions(limit: nil, since: nil, until_time: nil)
+        scan_cap = limit || store.limits.analytics_max_scan_sessions
+        seen = {}
+        store.each_session_events(limit: scan_cap, since: since, until_time: until_time) do |summary, window_id, events|
+          seen[summary[:session_id]] = true
+          yield summary, window_id, events
+        end
+        [seen.size, seen.size >= scan_cap]
+      end
+
+      def duration_ms(summary)
+        first = summary[:first_event_at]
+        last = summary[:last_event_at]
+        return nil unless first && last
+
+        (last - first).abs
+      end
+
+      def meta_event(events)
+        events.find { |event| event["type"] == META && event["data"].is_a?(Hash) }
+      end
+
+      # Splits a window's events into per-page segments on Meta href boundaries
+      # (one non-SPA window spans every page). Yields [url, segment_events,
+      # anchor_ts]; consecutive same-href Metas (same-URL reloads) stay in one
+      # segment. anchor_ts is the WINDOW's first timestamp for every segment:
+      # replay deep-links (?t=offset) are window-relative, never segment-local.
+      def each_page_segment(events)
+        return if events.empty?
+
+        anchor_ts = events.first&.fetch("timestamp", nil)
+
+        boundaries = [] # [start_index, url] per href change
+        events.each_with_index do |event, index|
+          url = meta_href(event)
+          next unless url
+
+          boundaries << [index, url] if boundaries.empty? || boundaries.last[1] != url
+        end
+
+        if boundaries.empty?
+          yield nil, events, anchor_ts
+          return
+        end
+
+        boundaries.each_with_index do |(start, url), i|
+          start = 0 if i.zero? # pre-first-Meta events belong to the first page
+          stop = boundaries[i + 1]&.first || events.size
+          yield url, events[start...stop], anchor_ts
+        end
+      end
+
+      def meta_href(event)
+        return nil unless event.is_a?(Hash) && event["type"] == META
+
+        data = event["data"]
+        href = data.is_a?(Hash) ? data["href"] : nil
+        (href.is_a?(String) && !href.empty?) ? href : nil
+      end
+    end
+  end
+end

data/lib/sentiero/analytics/bounded.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Sentiero
+  module Analytics
+    # Cap primitives shared by the compute-on-read collectors/analyzers, which
+    # bound memory during a scan. Both leave the collection unchanged once full,
+    # so the caller flips its own `@capped` flag on a false/nil return.
+    module Bounded
+      # Counter cap: bump counts[key] (a Hash defaulting to 0), adding a NEW key
+      # only while under `cap` (nil = unbounded). Returns true if counted, false
+      # if the cap dropped it.
+      def bounded_increment(counts, key, cap, by: 1)
+        return false unless counts.key?(key) || cap.nil? || counts.size < cap
+
+        counts[key] += by
+        true
+      end
+
+      # Slot cap: the existing entry, a freshly built one (yielded, while under
+      # `cap`), or nil when the store already holds `cap` distinct keys.
+      def bounded_fetch(store, key, cap)
+        return store[key] if store.key?(key)
+        return nil if cap && store.size >= cap
+
+        store[key] = yield
+      end
+    end
+  end
+end

data/lib/sentiero/analytics/browser_event_discovery.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require_relative "analyzer"
+
+module Sentiero
+  module Analytics
+    # Cross-session newest-first listing of browser custom events (rrweb type==5)
+    # excluding the "error" tag (errors have ErrorDiscovery). Each row carries
+    # session/window + offset(ms) from window start for replay deep-links (?t=).
+    class BrowserEventDiscovery < Analyzer
+      ERROR_TAG = "error"
+
+      MAX_ROWS = 500
+
+      # Trim cap during the scan so a busy store can't balloon memory. Mid-scan
+      # trimming is safe: it keeps the globally-newest seen, and newer events
+      # from later sessions still get added and survive the next trim.
+      ACCUMULATION_LIMIT = MAX_ROWS * 4
+
+      def recent_events(since: nil, until_time: nil)
+        rows = []
+        truncated = false
+
+        _scanned, hit_cap = scan_sessions(since: since, until_time: until_time) do |summary, window_id, events|
+          anchor = events.first&.fetch("timestamp", nil)
+          events.each do |event|
+            next unless browser_event?(event)
+
+            rows << build_row(summary, window_id, anchor, event)
+          end
+
+          next unless rows.size > ACCUMULATION_LIMIT
+
+          rows.sort_by! { |r| -(r[:timestamp] || 0) }
+          rows = rows.first(MAX_ROWS)
+          truncated = true
+        end
+
+        rows.sort_by! { |r| -(r[:timestamp] || 0) }
+        {
+          rows: rows.first(MAX_ROWS),
+          was_truncated: truncated || hit_cap || rows.size > MAX_ROWS
+        }
+      end
+
+      private
+
+      def browser_event?(event)
+        return false unless event["type"] == CUSTOM
+
+        data = event["data"]
+        data.is_a?(Hash) && data["tag"] != ERROR_TAG
+      end
+
+      def build_row(summary, window_id, anchor, event)
+        data = event["data"] || {}
+        ts = event["timestamp"]
+        payload = data["payload"]
+        {
+          name: data["tag"].to_s,
+          session_id: summary[:session_id],
+          window_id: window_id,
+          timestamp: ts,
+          offset_ms: offset_ms(anchor, ts),
+          payload: payload.is_a?(Hash) ? payload : nil
+        }
+      end
+    end
+  end
+end

data/lib/sentiero/analytics/collectors/click_collector.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require_relative "../events"
+require_relative "../bounded"
+
+module Sentiero
+  module Analytics
+    # Per-URL click density grid + element selectors across page segments. A
+    # click's viewport y becomes a page coordinate by adding the latest scroll
+    # offset, normalized against the estimated page height (deepest scroll +
+    # viewport); x by viewport width. Both bucket into a GRID_SIZE x GRID_SIZE grid.
+    class ClickCollector
+      include Events
+      include Bounded
+
+      MOUSE_CLICK = 2
+
+      # Carries the clicked element's CSS selector; rrweb's own click event only
+      # references an internal node id, not a stable selector.
+      CLICK_TAG = "__click"
+
+      # Grid resolution per axis.
+      GRID_SIZE = 20
+
+      attr_reader :total, :buckets, :selectors, :capped
+
+      def initialize(max_selectors: nil)
+        @max_selectors = max_selectors
+        @buckets = Hash.new(0)
+        @selectors = Hash.new(0)
+        @total = 0
+        @capped = false
+      end
+
+      # Returns clicks added, or nil when the segment has no usable viewport
+      # (callers branch on the nil).
+      def collect(segment)
+        viewport = viewport_size(segment)
+        return nil unless viewport
+
+        page_height = estimate_page_height(segment, viewport)
+        scroll_y = 0
+        added = 0
+
+        segment.each do |event|
+          scroll_y = 0 if event["type"] == META
+          if (y = document_scroll_y(event))
+            scroll_y = y
+          end
+          if click?(event)
+            data = event["data"]
+            @buckets[bucket(data["x"], data["y"] + scroll_y, viewport, page_height)] += 1
+            added += 1
+          end
+          tally_selector(event)
+        end
+
+        @total += added
+        added
+      end
+
+      private
+
+      def viewport_size(events)
+        meta = events.find { |event| event["type"] == META && event["data"].is_a?(Hash) }
+        return nil unless meta
+
+        width = meta.dig("data", "width")
+        height = meta.dig("data", "height")
+        return nil unless width.is_a?(Numeric) && height.is_a?(Numeric)
+        return nil unless width > 0 && height > 0
+
+        {width: width, height: height}
+      end
+
+      def estimate_page_height(segment, viewport)
+        max_scroll = 0
+        segment.each do |event|
+          y = document_scroll_y(event)
+          max_scroll = y if y && y > max_scroll
+        end
+        max_scroll + viewport[:height]
+      end
+
+      def document_scroll_y(event)
+        return nil unless event["type"] == INCREMENTAL
+
+        data = event["data"]
+        return nil unless data.is_a?(Hash) && data["source"] == SOURCE_SCROLL
+
+        id = data["id"]
+        return nil unless id.nil? || id == 1
+
+        y = data["y"]
+        (y.is_a?(Numeric) && y >= 0) ? y : nil
+      end
+
+      def click?(event)
+        return false unless event["type"] == INCREMENTAL
+
+        data = event["data"]
+        return false unless data.is_a?(Hash)
+
+        data["source"] == SOURCE_MOUSE_INTERACTION &&
+          data["type"] == MOUSE_CLICK &&
+          data["x"].is_a?(Numeric) &&
+          data["y"].is_a?(Numeric)
+      end
+
+      def bucket(x, page_y, viewport, page_height)
+        [
+          bucket_index(x, viewport[:width]),
+          bucket_index(page_y, page_height)
+        ]
+      end
+
+      def bucket_index(value, axis_length)
+        index = (value.to_f / axis_length * GRID_SIZE).floor
+        index.clamp(0, GRID_SIZE - 1)
+      end
+
+      def tally_selector(event)
+        return unless event["type"] == CUSTOM
+
+        data = event["data"]
+        return unless data.is_a?(Hash) && data["tag"] == CLICK_TAG
+
+        selector = data.dig("payload", "selector")
+        return unless selector.is_a?(String) && !selector.empty?
+
+        @capped = true unless bounded_increment(@selectors, selector, @max_selectors)
+      end
+    end
+  end
+end

data/lib/sentiero/analytics/collectors/custom_tag_collector.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require_relative "../events"
+require_relative "../bounded"
+require_relative "../stats"
+
+module Sentiero
+  module Analytics
+    # Custom-event tag tally across page segments. The single definition of
+    # which tags are "internal" and how the rest are counted and ranked.
+    class CustomTagCollector
+      include Events
+      include Bounded
+      include Stats
+
+      # Recorder-internal annotations (__perf, __click, …); never on the panel.
+      INTERNAL_TAG_PREFIX = "__"
+      # The JS-error tag is also internal — it has its own panel.
+      ERROR_TAG = "error"
+      MAX_CUSTOM_TAGS = 200
+
+      attr_reader :tags, :capped
+
+      # max_tags: nil unbounded; an Integer caps distinct tags, flipping #capped.
+      def initialize(max_tags: nil)
+        @max_tags = max_tags
+        @tags = Hash.new(0)
+        @capped = false
+      end
+
+      def internal_tag?(tag)
+        tag.start_with?(INTERNAL_TAG_PREFIX) || tag == ERROR_TAG
+      end
+
+      # Returns true when counted, false when internal or capped — callers gate
+      # per-tag side-effects on this.
+      def tally(tag)
+        return false if internal_tag?(tag)
+
+        counted = bounded_increment(@tags, tag, @max_tags)
+        @capped = true unless counted
+        counted
+      end
+
+      def collect(segment)
+        segment.each do |event|
+          next unless event["type"] == CUSTOM
+
+          tag = event.dig("data", "tag")
+          next unless tag.is_a?(String) && !tag.empty?
+
+          tally(tag)
+        end
+      end
+
+      def top(n)
+        top_counts(@tags, limit: n).map { |tag, count| {tag: tag, count: count} }
+      end
+    end
+  end
+end

data/lib/sentiero/analytics/collectors/error_collector.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require_relative "../events"
+require_relative "../stats"
+require_relative "../bounded"
+
+module Sentiero
+  module Analytics
+    # Per-URL error grouping across page segments. Groups JS errors by a
+    # normalized key (see group_key) so messages differing only by an
+    # id/count/line number collapse into one row. Each occurrence records
+    # offset_ms from the window's first event so the UI can deep-link via ?t=.
+    # The three helpers are class methods so ErrorDiscovery can reuse them with
+    # its own group shape without instantiating an accumulator.
+    class ErrorCollector
+      include Events
+      include Stats
+      include Bounded
+
+      ERROR_TAG = "error"
+      MAX_KEY_LENGTH = 200
+
+      attr_reader :groups, :capped
+
+      # Integer caps distinct groups (flips #capped) / occurrences per group; nil unbounded.
+      def initialize(max_groups: nil, max_occurrences: nil)
+        @max_groups = max_groups
+        @max_occurrences = max_occurrences
+        @groups = {}
+        @capped = false
+      end
+
+      # anchor is the window's first-event timestamp; offset_ms is relative to it
+      # so replay ?t= deep-links stay consistent across segments of the window.
+      def collect(segment, session_id:, window_id:, anchor:)
+        segment.each do |event|
+          next unless self.class.error_event?(event)
+
+          message = self.class.extract_message(event)
+          key = self.class.group_key(message)
+
+          group = bounded_fetch(@groups, key, @max_groups) { {message: message, count: 0, occurrences: []} }
+          if group.nil?
+            @capped = true
+            next
+          end
+
+          group[:count] += 1
+
+          if @max_occurrences.nil? || group[:occurrences].size < @max_occurrences
+            group[:occurrences] << {
+              session_id: session_id,
+              window_id: window_id,
+              offset_ms: offset_ms(anchor, event["timestamp"])
+            }
+          end
+        end
+      end
+
+      def summarize
+        {
+          groups: @groups.values
+            .sort_by { |g| -g[:count] }
+            .map { |g| {message: g[:message], count: g[:count], occurrences: g[:occurrences]} },
+          total: @groups.values.sum { |g| g[:count] }
+        }
+      end
+
+      def self.error_event?(event)
+        return false unless event["type"] == CUSTOM
+
+        data = event["data"]
+        data.is_a?(Hash) && data["tag"] == ERROR_TAG
+      end
+
+      def self.group_key(message)
+        message.lines.first.to_s.strip.gsub(/\d+/, "#")[0, MAX_KEY_LENGTH]
+      end
+
+      def self.extract_message(event)
+        payload = event.dig("data", "payload")
+        message = payload.is_a?(Hash) ? payload["message"] : nil
+        return "Unknown error" if message.nil? || message.to_s.strip.empty?
+
+        message.to_s
+      end
+    end
+  end
+end

data/lib/sentiero/analytics/collectors/form_collector.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+require_relative "../events"
+require_relative "../bounded"
+
+module Sentiero
+  module Analytics
+    # Per-URL form interaction math. The single definition of what an "input"
+    # and a "submit" are in rrweb terms. Two completion semantics for two callers:
+    #   #completed_count — sessions with inputs where EVERY input segment was
+    #     submitted (strict; one abandoned segment disqualifies).
+    #   #submitted_count — sessions with ANY __form_submit event, regardless of
+    #     input timing.
+    class FormCollector
+      include Events
+      include Bounded
+
+      # Recorder tag for a document-level form submit.
+      SUBMIT_TAG = "__form_submit"
+
+      # Output cap for the drop-off table.
+      DROP_OFF_LIMIT = 50
+
+      attr_reader :total_submits, :capped
+
+      # max_fields: nil unbounded; an Integer caps the fields hash, flipping #capped.
+      def initialize(max_fields: nil)
+        @max_fields = max_fields
+        @total_submits = 0
+        @fields = {}            # [url, node_id] => field-stats hash
+        @drop_off = Hash.new(0) # [url, node_id] => abandon count
+        @started = {}           # session_id => true  (≥1 input event seen)
+        @submitted = {}         # session_id => true  (≥1 submit event, any segment)
+        @abandoned = {}         # session_id => true  (≥1 input segment not submitted)
+        @capped = false
+      end
+
+      # Returns the count of input events found. labels: {node_id => label} from
+      # the segment's DOM snapshot; {} omits them.
+      def collect(session_id, url, segment, labels: {})
+        @total_submits += segment.count { |e| submit?(e) }
+        @submitted[session_id] = true if segment.any? { |e| submit?(e) }
+
+        inputs = segment.select { |e| input?(e) }
+        return 0 if inputs.empty?
+
+        @started[session_id] = true
+        record_fields(session_id, url, inputs, labels)
+
+        first_input_at = inputs.first["timestamp"]
+        unless segment_submitted?(segment, first_input_at)
+          @abandoned[session_id] = true
+          @drop_off[[url, node_id(inputs.last)]] += 1
+        end
+
+        inputs.size
+      end
+
+      def started_count
+        @started.size
+      end
+
+      # Counts a submit on the target URL even when inputs landed on a prior segment.
+      def submitted_count
+        @submitted.size
+      end
+
+      # Sessions with inputs where NO input segment was abandoned; a submit on a
+      # later page never masks an abandonment.
+      def completed_count
+        @started.count { |id, _| !@abandoned.key?(id) }
+      end
+
+      def summarize_fields(started, include_labels: false)
+        @fields
+          .sort_by { |(url, id), stats| [-stats[:sessions], url.to_s, id] }
+          .map do |(url, id), stats|
+            row = {}
+            row[:field_id] = id
+            row[:label] = stats[:label] if include_labels
+            row[:url] = url
+            row[:sessions] = stats[:sessions]
+            row[:completion_rate] = started.zero? ? 0.0 : stats[:sessions].to_f / started
+            row[:avg_time_to_fill_ms] = stats[:units].zero? ? 0.0 : stats[:total_fill_ms] / stats[:units]
+            row[:total_refills] = stats[:total_refills]
+            row
+          end
+      end
+
+      def summarize_drop_off(include_labels: false)
+        @drop_off
+          .sort_by { |(url, id), count| [-count, url.to_s, id] }
+          .first(DROP_OFF_LIMIT)
+          .map do |(url, id), count|
+            row = {}
+            row[:field_id] = id
+            row[:label] = @fields.key?([url, id]) ? @fields[[url, id]][:label] : nil if include_labels
+            row[:url] = url
+            row[:count] = count
+            row
+          end
+      end
+
+      private
+
+      def input?(event)
+        return false unless event["type"] == INCREMENTAL
+
+        data = event["data"]
+        data.is_a?(Hash) && data["source"] == SOURCE_INPUT && node_id(event)
+      end
+
+      def node_id(event)
+        id = event.dig("data", "id")
+        id.is_a?(Integer) ? id : nil
+      end
+
+      def submit?(event)
+        event["type"] == CUSTOM && event.dig("data", "tag") == SUBMIT_TAG
+      end
+
+      # A segment counts as submitted only when a __form_submit lands at or
+      # after the first input; an earlier submit belongs to a prior interaction
+      # (counting it resurrects "navigating away counts as submitting").
+      def segment_submitted?(segment, first_input_at)
+        segment.any? do |event|
+          next false unless submit?(event)
+
+          ts = event["timestamp"]
+          !first_input_at.is_a?(Numeric) || (ts.is_a?(Numeric) && ts >= first_input_at)
+        end
+      end
+
+      # Keyed by [url, node_id]: rrweb node ids reset on every full-page load, so
+      # the url scope keeps unrelated fields from conflating across pages.
+      def record_fields(session_id, url, inputs, labels)
+        inputs.group_by { |e| [url, node_id(e)] }.each do |key, field_inputs|
+          stats = bounded_fetch(@fields, key, @max_fields) do
+            {sessions: 0, units: 0, total_fill_ms: 0.0, total_refills: 0, last_session: nil, label: nil}
+          end
+          if stats.nil?
+            @capped = true
+            next
+          end
+          stats[:sessions] += 1 unless stats[:last_session] == session_id
+          stats[:last_session] = session_id
+          stats[:units] += 1
+          timestamps = field_inputs.map { |e| e["timestamp"] }
+          stats[:total_fill_ms] += (timestamps.max - timestamps.min).to_f
+          stats[:total_refills] += field_inputs.size - 1
+          stats[:label] ||= labels[node_id(field_inputs.first)]
+        end
+      end
+    end
+  end
+end