sentiero 1.0.0.alpha1

data/lib/sentiero/analytics/exporter.rb

@@ -0,0 +1,242 @@
+# frozen_string_literal: true
+
+require_relative "analyzer"
+require_relative "stats_aggregator"
+require_relative "error_discovery"
+require_relative "heatmap_analyzer"
+require_relative "scroll_depth_analyzer"
+require_relative "form_analyzer"
+require_relative "web_vitals_analyzer"
+require_relative "../user_agent"
+
+module Sentiero
+  module Analytics
+    # Builds tabular datasets for CSV/JSON export. Each dataset is a
+    # {headers:, rows:} table the web layer serializes without re-deriving.
+    class Exporter < Analyzer
+      DATASETS = {
+        "sessions" => "Session list",
+        "errors" => "Error list",
+        "browser_events" => "Browser Events (rrweb)",
+        "problems" => "Problems",
+        "server_events" => "Server Events",
+        "stats" => "Aggregate stats",
+        "heatmap" => "Heatmap data",
+        "scroll" => "Scroll-depth data",
+        "forms" => "Form analytics",
+        "web_vitals" => "Web Vitals"
+      }.freeze
+
+      def dataset?(name)
+        DATASETS.key?(name)
+      end
+
+      def table(name, since: nil, until_time: nil)
+        @since = since
+        @until_time = until_time
+        send("build_#{name}")
+      end
+
+      private
+
+      attr_reader :since, :until_time
+
+      def scan_cap
+        store.limits.analytics_max_scan_sessions
+      end
+
+      def build_sessions
+        headers = %w[session_id created_at first_event_at last_event_at
+          duration_ms event_count url referrer browser device has_errors]
+
+        rows = store.list_sessions(limit: scan_cap, since: since, until_time: until_time).map do |summary|
+          metadata = summary[:metadata] || {}
+          user_agent = metadata["userAgent"]
+          [
+            summary[:session_id],
+            summary[:created_at],
+            summary[:first_event_at],
+            summary[:last_event_at],
+            duration_ms(summary),
+            summary[:event_count],
+            metadata["url"],
+            metadata["referrer"],
+            UserAgent.browser(user_agent),
+            UserAgent.device(user_agent),
+            metadata["has_errors"] == true
+          ]
+        end
+
+        {headers: headers, rows: rows}
+      end
+
+      def build_errors
+        headers = %w[message source line count last_seen_at session_id window_id offset_ms]
+
+        rows = ErrorDiscovery.new(store).grouped_errors(since: since, until_time: until_time)[:groups].flat_map do |group|
+          group[:occurrences].map do |occ|
+            [
+              group[:message],
+              group[:source],
+              group[:line],
+              group[:count],
+              group[:last_seen_at],
+              occ[:session_id],
+              occ[:window_id],
+              occ[:offset_ms]
+            ]
+          end
+        end
+
+        {headers: headers, rows: rows}
+      end
+
+      def build_browser_events
+        headers = %w[session_id window_id timestamp tag]
+        rows = []
+
+        store.each_session_events(limit: scan_cap, since: since, until_time: until_time) do |summary, window_id, events|
+          events.each do |event|
+            next unless event["type"] == CUSTOM
+            tag = event.dig("data", "tag")
+            rows << [summary[:session_id], window_id, event["timestamp"], tag]
+          end
+        end
+
+        {headers: headers, rows: rows}
+      end
+
+      def build_problems
+        headers = %w[id fingerprint project exception_class title count status first_seen last_seen]
+
+        rows = store.list_problems(project: nil, limit: scan_cap, since: since, until_time: until_time).map do |problem|
+          [
+            problem[:id],
+            # A problem's id is its fingerprint; stores don't expose a separate key.
+            problem[:id],
+            problem[:project],
+            problem[:exception_class],
+            problem[:title],
+            problem[:count],
+            problem[:status],
+            problem[:first_seen],
+            problem[:last_seen]
+          ]
+        end
+
+        {headers: headers, rows: rows}
+      end
+
+      def build_server_events
+        headers = %w[id project name level session_id timestamp payload]
+
+        # list_server_events' `after` is an exclusive cursor; filter the range
+        # ourselves so `since`/`until` are both inclusive, like every other dataset.
+        events = store.list_server_events(project: nil, limit: scan_cap)
+        events = events.select { |event| event["timestamp"].to_f >= since } if since
+        events = events.select { |event| event["timestamp"].to_f <= until_time } if until_time
+
+        rows = events.map do |event|
+          [
+            event["id"],
+            event["project"],
+            event["name"],
+            event["level"],
+            event["session_id"],
+            event["timestamp"],
+            event["payload"].is_a?(Hash) ? event["payload"].to_json : event["payload"].to_s
+          ]
+        end
+
+        {headers: headers, rows: rows}
+      end
+
+      def build_stats
+        stats = StatsAggregator.new(store).aggregate(since: since, until_time: until_time)
+
+        headers = %w[metric value]
+        rows = [
+          ["total_sessions", stats[:total_sessions]],
+          ["total_events", stats[:total_events]],
+          ["avg_duration_ms", stats[:avg_duration_ms]]
+        ]
+        stats[:browser_distribution].each { |browser, count| rows << ["browser:#{browser}", count] }
+        stats[:device_distribution].each { |device, count| rows << ["device:#{device}", count] }
+        stats[:custom_event_tags].each { |tag, count| rows << ["custom_event:#{tag}", count] }
+
+        {headers: headers, rows: rows}
+      end
+
+      def build_heatmap
+        headers = %w[url selector count]
+        rows = HeatmapAnalyzer.new(store).build_heatmap_table(since: since, until_time: until_time).flat_map do |url, elements|
+          elements.map { |element| [url, element[:selector], element[:count]] }
+        end
+
+        {headers: headers, rows: rows}
+      end
+
+      def build_scroll
+        # Percentages are absolute depth vs. estimated page height (deepest
+        # scroll + viewport across sessions).
+        headers = %w[url session_count avg_depth_px avg_depth_pct page_height_px p50_pct p75_pct p90_pct]
+
+        rows = ScrollDepthAnalyzer.new(store).analyze(since: since, until_time: until_time)[:pages].map do |url, page|
+          folds = page[:fold_lines]
+          [
+            url,
+            page[:session_count],
+            page[:avg_depth_px],
+            page[:avg_depth_pct],
+            page[:page_height_px],
+            folds[:p50],
+            folds[:p75],
+            folds[:p90]
+          ]
+        end
+
+        {headers: headers, rows: rows}
+      end
+
+      def build_web_vitals
+        headers = %w[url metric p50 p75 p90 samples good_count needs_improvement_count poor_count]
+
+        pages = WebVitalsAnalyzer.new(store).analyze(since: since, until_time: until_time)[:pages]
+        rows = pages.sort_by { |url, _page| url }.flat_map do |url, page|
+          page[:metrics].map do |metric, m|
+            ratings = m[:ratings]
+            [
+              url,
+              metric,
+              m[:p50],
+              m[:p75],
+              m[:p90],
+              m[:samples],
+              ratings.fetch("good", 0),
+              ratings.fetch("needs-improvement", 0),
+              ratings.fetch("poor", 0)
+            ]
+          end
+        end
+
+        {headers: headers, rows: rows}
+      end
+
+      def build_forms
+        headers = %w[field_id sessions completion_rate avg_time_to_fill_ms total_refills]
+
+        rows = FormAnalyzer.new(store).analyze(since: since, until_time: until_time)[:fields].map do |field|
+          [
+            field[:field_id],
+            field[:sessions],
+            field[:completion_rate],
+            field[:avg_time_to_fill_ms],
+            field[:total_refills]
+          ]
+        end
+
+        {headers: headers, rows: rows}
+      end
+    end
+  end
+end

data/lib/sentiero/analytics/form_analyzer.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+require_relative "analyzer"
+require_relative "collectors/form_collector"
+
+module Sentiero
+  module Analytics
+    # Cross-session form analysis built from interaction patterns: rrweb input
+    # events (incremental, type 3 / source 5) carry the touched node's id and
+    # timing but never values, so this works with maskAllInputs enabled.
+    #
+    # Attribution is per page: each session's windows are merged, ordered by
+    # time, and split into page segments on Meta-href boundaries (the shared
+    # Analyzer#each_page_segment mechanism). A (session, page) unit "starts" a
+    # form when the segment contains input events, and is "completed" only
+    # when a __form_submit custom event — emitted by the recorder's
+    # capture-phase document submit listener (config.track_forms) — lands in
+    # the same segment at or after the first input. A session counts as
+    # completed when EVERY page it interacted on was submitted, so a genuine
+    # submit elsewhere (a later todo add) can no longer mask an abandonment
+    # (the signup form it walked away from).
+    #
+    # CAPTURE-VERSION NOTE: submits are counted ONLY from __form_submit
+    # events. Windows recorded before that capture existed (or with
+    # track_forms off) carry none and intentionally report ZERO submits —
+    # falling back to counting bare Meta/navigation events would resurrect
+    # the "navigating away counts as submitting" defect (product review
+    # P1.4/D4: 100% shown where the funnel proved 50%).
+    #
+    # Fields are keyed per (page URL, node id) so the same rrweb node id on
+    # two different pages (ids reset every full-page load) no longer
+    # conflates two unrelated fields. Per field it reports touch rate
+    # (fraction of interacting sessions), aggregate time-to-fill, and re-fill
+    # frequency, plus a drop-off table (the last field touched in each
+    # abandoned page segment).
+    #
+    # Compute-on-read: a full scan of Store#each_session_events up to the
+    # store's limits.analytics_max_scan_sessions — no fact-extraction tables.
+    #
+    # Per-segment math (input recognition, field accumulation, drop-off,
+    # submit detection, and session-level started/completed semantics) lives in
+    # FormCollector so PageReportAnalyzer can share it without duplication.
+    class FormAnalyzer < Analyzer
+      # rrweb EventType.FullSnapshot and NodeType.Element — used to read field
+      # identity (name/id/type) from the DOM snapshot for human field labels.
+      FULL_SNAPSHOT = 2
+      ELEMENT_NODE = 2
+      FORM_CONTROL_TAGS = %w[input select textarea].freeze
+
+      # Aggregates form interactions across sessions. Returns per-field stats,
+      # the drop-off table, form-level completion rate, the raw submit count,
+      # and whether the scan was capped. since/until_time (epoch seconds)
+      # bound the scan at the store level.
+      def analyze(limit: nil, since: nil, until_time: nil)
+        scan_cap = limit || store.limits.analytics_max_scan_sessions
+        collector = FormCollector.new  # unbounded: no per-URL field cap here
+
+        sessions = merge_windows(scan_cap, since, until_time)
+        sessions.each { |session_id, session| analyze_session(collector, session_id, session) }
+
+        started = collector.started_count
+        {
+          sessions_with_form_interaction: started,
+          sessions_completed: collector.completed_count,
+          completion_rate: ratio(collector.completed_count, started),
+          total_submits: collector.total_submits,
+          fields: collector.summarize_fields(started, include_labels: true),
+          drop_off_fields: collector.summarize_drop_off(include_labels: true),
+          was_truncated: sessions.size >= scan_cap
+        }
+      end
+
+      private
+
+      # Each session may span several windows; each_session_events yields once
+      # per window. Concatenate a session's windows so it is analyzed once and
+      # its cross-window form interactions are seen together.
+      def merge_windows(scan_cap, since, until_time)
+        sessions = Hash.new { |h, id| h[id] = {events: []} }
+
+        store.each_session_events(limit: scan_cap, since: since, until_time: until_time) do |summary, _window_id, events|
+          sessions[summary[:session_id]][:events].concat(events)
+        end
+
+        sessions
+      end
+
+      # Walks one session's page segments, feeding each to the shared collector.
+      # Labels are built per-segment from the DOM snapshot so field identities
+      # (name/id/type) survive same-URL reloads correctly.
+      def analyze_session(collector, session_id, session)
+        events = order_by_time(session[:events])
+        each_page_segment(events) do |url, segment, _anchor|
+          collector.collect(session_id, url, segment, labels: field_labels(segment))
+        end
+      end
+
+      # Concatenated windows interleave in time, so order by timestamp to keep
+      # segmentation and first-input/submit ordering correct. Events without a
+      # numeric timestamp sort to the front so they never count as "after" input.
+      def order_by_time(events)
+        events.sort_by { |event| event["timestamp"].is_a?(Numeric) ? event["timestamp"] : -Float::INFINITY }
+      end
+
+      # Maps node id => human field label from the full DOM snapshots (rrweb
+      # type 2) in this page segment. rrweb input events carry only the node id;
+      # the snapshot is the only place the field's name/id/type live, and node
+      # ids are scoped to a page load, so this is built per segment. Attributes
+      # only — never values. Returns {} when the segment has no snapshot
+      # (incremental-only windows fall back to nil labels in the output).
+      def field_labels(segment)
+        segment.each_with_object({}) do |event, labels|
+          next unless event["type"] == FULL_SNAPSHOT
+          collect_field_labels(event.dig("data", "node"), labels)
+        end
+      end
+
+      def collect_field_labels(node, labels)
+        return unless node.is_a?(Hash)
+        if node["type"] == ELEMENT_NODE && FORM_CONTROL_TAGS.include?(node["tagName"])
+          id = node["id"]
+          labels[id] = field_label(node["attributes"] || {}, node["tagName"]) if id.is_a?(Integer)
+        end
+        children = node["childNodes"]
+        children.each { |child| collect_field_labels(child, labels) } if children.is_a?(Array)
+      end
+
+      # Prefer the field's name, then its DOM id, then its type; append the
+      # input type for context when a named text-like field exists, and never
+      # emit values.
+      def field_label(attrs, tag)
+        base = present(attrs["name"]) || present(attrs["id"])
+        type = present(attrs["type"])
+        if base
+          (tag == "input" && type) ? "#{base} (#{type})" : base
+        elsif tag == "select"
+          "select"
+        else
+          type || tag
+        end
+      end
+
+      def present(value)
+        value if value.is_a?(String) && !value.empty?
+      end
+
+      def ratio(numerator, denominator)
+        return 0 if denominator.zero?
+        numerator.to_f / denominator
+      end
+    end
+  end
+end

data/lib/sentiero/analytics/frustration/detectors.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+require_relative "../events"
+
+module Sentiero
+  module Analytics
+    module Frustration
+      # Pure, self-contained frustration detectors — Ruby ports of the JS
+      # detectors (frontend/src/dashboard/frustration.js), pinned by ported
+      # tests so the two can't drift. FrustrationAnalyzer layers cross-session
+      # aggregation and de-noising on top of this raw detector output.
+      module Detectors
+        # Detector thresholds — verbatim from frontend/src/dashboard/frustration.js.
+        RAGE_WINDOW_MS = 500          # max span of a rage cluster
+        RAGE_COORD_TOLERANCE_PX = 10  # max px from the first click
+        RAGE_MIN_CLICKS = 3           # min clicks to count as rage
+        DEAD_WINDOW_MS = 500          # response deadline for a dead click
+
+        MOUSE_CLICK = 2     # rrweb MouseInteraction click subtype
+        SOURCE_MUTATION = 0 # rrweb IncrementalSource.Mutation
+
+        module_function
+
+        def detect_rage_clicks(events)
+          return [] unless events.is_a?(Array)
+
+          clicks = events.select { |event| click?(event) }
+          return [] if clicks.size < RAGE_MIN_CLICKS
+
+          out = []
+          cluster_start = 0
+          (1..clicks.size).each do |i|
+            prev = clicks[i - 1]
+            cur = clicks[i]
+            anchor = clicks[cluster_start]
+            continues = cur &&
+              cur["timestamp"] - prev["timestamp"] <= RAGE_WINDOW_MS &&
+              cur["timestamp"] - anchor["timestamp"] <= RAGE_WINDOW_MS &&
+              (cur["data"]["x"] - anchor["data"]["x"]).abs <= RAGE_COORD_TOLERANCE_PX &&
+              (cur["data"]["y"] - anchor["data"]["y"]).abs <= RAGE_COORD_TOLERANCE_PX
+
+            next if continues
+
+            count = i - cluster_start
+            if count >= RAGE_MIN_CLICKS
+              out << {
+                subtype: "rage_click",
+                timestamp: anchor["timestamp"],
+                count: count,
+                x: anchor["data"]["x"],
+                y: anchor["data"]["y"],
+                member_timestamps: clicks[cluster_start...i].map { |c| c["timestamp"] },
+                event: anchor
+              }
+            end
+            cluster_start = i
+          end
+          out
+        end
+
+        # Clicks with no page response within DEAD_WINDOW_MS.
+        def detect_dead_clicks(events)
+          return [] unless events.is_a?(Array)
+
+          out = []
+          events.each_with_index do |event, i|
+            next unless click?(event)
+            click_ts = event["timestamp"]
+            deadline = click_ts + DEAD_WINDOW_MS
+
+            responded = false
+            (i + 1...events.size).each do |j|
+              ts = events[j].is_a?(Hash) ? events[j]["timestamp"] : nil
+              next unless ts.is_a?(Numeric)
+              break if ts > deadline
+              if ts > click_ts && response?(events[j])
+                responded = true
+                break
+              end
+            end
+
+            unless responded
+              out << {
+                subtype: "dead_click",
+                timestamp: click_ts,
+                x: event["data"]["x"],
+                y: event["data"]["y"],
+                elapsed: DEAD_WINDOW_MS,
+                event: event
+              }
+            end
+          end
+          out
+        end
+
+        # Combines both detectors (clicks absorbed into a rage cluster are not
+        # also reported as dead), sorted by offset from the window's first event.
+        def detect_frustration_events(events)
+          return [] unless events.is_a?(Array) && !events.empty?
+          first = events.first
+          return [] unless first.is_a?(Hash) && first["timestamp"].is_a?(Numeric)
+          first_ts = first["timestamp"]
+
+          rage = detect_rage_clicks(events)
+          dead = detect_dead_clicks(events)
+
+          rage_timestamps = {}
+          rage.each do |r|
+            (r[:member_timestamps] || [r[:timestamp]]).each { |t| rage_timestamps[t] = true }
+          end
+
+          combined = rage + dead.reject { |d| rage_timestamps[d[:timestamp]] }
+
+          combined
+            .map do |entry|
+              {
+                category: "frustration",
+                subtype: entry[:subtype],
+                timestamp: entry[:timestamp],
+                offset: entry[:timestamp] - first_ts,
+                count: entry[:count],
+                elapsed: entry[:elapsed],
+                x: entry[:x],
+                y: entry[:y],
+                event: entry[:event]
+              }
+            end
+            .each_with_index.sort_by { |entry, i| [entry[:offset], i] }
+            .map { |entry, _i| entry }
+        end
+
+        # rrweb left-click mouse-interaction carrying coordinates (mirrors JS isClick).
+        def click?(event)
+          return false unless event.is_a?(Hash)
+          return false unless event["type"] == Events::INCREMENTAL
+          return false unless event["timestamp"].is_a?(Numeric)
+          data = event["data"]
+          data.is_a?(Hash) &&
+            data["source"] == Events::SOURCE_MOUSE_INTERACTION &&
+            data["type"] == MOUSE_CLICK &&
+            data["x"].is_a?(Numeric) &&
+            data["y"].is_a?(Numeric)
+        end
+
+        # Page responded to a click: DOM mutation, input change, or meta/navigation
+        # (mirrors JS isResponse).
+        def response?(event)
+          return false unless event.is_a?(Hash) && event["data"]
+          return true if event["type"] == Events::META
+          return false unless event["type"] == Events::INCREMENTAL
+          data = event["data"]
+          return false unless data.is_a?(Hash)
+          data["source"] == SOURCE_MUTATION || data["source"] == Events::SOURCE_INPUT
+        end
+      end
+    end
+  end
+end