sentiero 1.0.0.alpha1

data/lib/sentiero/analytics/collectors/frustration_collector.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require_relative "../events"
+require_relative "../bounded"
+
+module Sentiero
+  module Analytics
+    # Per-segment frustration attribution. Attributes incidents to segments by
+    # object identity (e.equal?(incident[:event])) so a same-millisecond Meta
+    # boundary cannot mis-attribute one.
+    #
+    # IMPORTANT: works on the RAW detector output (before refine_incidents
+    # de-noise), so dead_count may EXCEED /analytics/frustration for the same
+    # URL. Intentional: completeness over precision (no rages a de-noise rule
+    # might withdraw are missed).
+    class FrustrationCollector
+      include Events
+      include Bounded
+
+      # Recorder tag carrying the clicked element's CSS selector.
+      CLICK_TAG = "__click"
+
+      attr_reader :rage_count, :dead_count, :selectors, :capped
+
+      def initialize(max_selectors: nil)
+        @max_selectors = max_selectors
+        @rage_count = 0
+        @dead_count = 0
+        @selectors = Hash.new(0)
+        @capped = false
+      end
+
+      # Returns the number attributed to this segment.
+      def collect(incidents, segment)
+        return 0 if incidents.empty?
+
+        attributed = 0
+        incidents.each do |incident|
+          next unless segment.any? { |e| e.equal?(incident[:event]) }
+
+          if incident[:subtype] == "rage_click"
+            @rage_count += 1
+            selector = nearest_click_selector(segment, incident[:timestamp])
+            if selector
+              @capped = true unless bounded_increment(@selectors, selector, @max_selectors)
+            end
+          else
+            @dead_count += 1
+          end
+
+          attributed += 1
+        end
+        attributed
+      end
+
+      private
+
+      # Nearest "__click" selector by timestamp. No distance ceiling — the
+      # segment is bounded to one page.
+      def nearest_click_selector(segment, timestamp)
+        nearest = nil
+        nearest_distance = nil
+        segment.each do |event|
+          next unless event["type"] == CUSTOM
+
+          data = event["data"]
+          next unless data.is_a?(Hash) && data["tag"] == CLICK_TAG
+
+          selector = data.dig("payload", "selector")
+          next unless selector.is_a?(String) && !selector.empty?
+
+          ts = event["timestamp"]
+          next unless ts.is_a?(Numeric)
+
+          distance = (ts - timestamp).abs
+          if nearest_distance.nil? || distance < nearest_distance
+            nearest_distance = distance
+            nearest = selector
+          end
+        end
+        nearest
+      end
+    end
+  end
+end

data/lib/sentiero/analytics/collectors/scroll_collector.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+require_relative "../events"
+require_relative "../stats"
+require_relative "../bounded"
+
+module Sentiero
+  module Analytics
+    # Per-URL scroll depth across page segments and windows.
+    #
+    # rrweb Metas carry the viewport height but NOT the document height, so page
+    # height is ESTIMATED as the deepest (max scroll + viewport) any sample
+    # reached — exact when somebody read to the end, a lower bound otherwise.
+    # Viewport-less samples fall back to pixels (no percentage derivable).
+    #
+    # Per window: #observe each segment, then #flush_window once to commit each
+    # URL's deepest segment as ONE sample.
+    class ScrollCollector
+      include Events
+      include Stats
+      include Bounded
+
+      DISTRIBUTION_BINS = %w[0-25 25-50 50-75 75-100].freeze
+
+      attr_reader :capped
+
+      # max_urls: nil unbounded; an Integer caps distinct URLs, flipping #capped.
+      def initialize(max_urls: nil)
+        @max_urls = max_urls
+        @samples_by_url = {} # url => [{max_y:, viewport_height:}, ...]
+        @window = {} # url => deepest segment depth in the current window
+        @capped = false
+      end
+
+      # Deepest segment per url wins; segments with no scroll are ignored.
+      def observe(url, segment)
+        depth = segment_depth(segment)
+        return unless depth
+
+        current = @window[url]
+        @window[url] = depth if current.nil? || depth[:max_y] > current[:max_y]
+      end
+
+      # One sample per (url, window): the deepest of the window's segments, then resets.
+      def flush_window
+        @window.each do |url, depth|
+          samples = bounded_fetch(@samples_by_url, url, @max_urls) { [] }
+          if samples.nil?
+            @capped = true
+            next
+          end
+          samples << depth
+        end
+        @window = {}
+      end
+
+      # nil when nothing was recorded for the URL.
+      def summarize(url)
+        samples = @samples_by_url[url]
+        return nil unless samples && !samples.empty?
+
+        summarize_samples(samples)
+      end
+
+      def pages
+        @samples_by_url.transform_values { |samples| summarize_samples(samples) }
+      end
+
+      private
+
+      def segment_depth(segment)
+        max_y = segment.filter_map { |event| scroll_y(event) }.max || 0
+        return nil unless max_y > 0
+
+        {max_y: max_y, viewport_height: viewport_height(segment)}
+      end
+
+      # Only the document scroll (node id nil or 1) counts as page depth; inner
+      # scroll containers (id > 1) would otherwise inflate it. Mirrors
+      # ClickCollector#document_scroll_y so both agree on "page scroll".
+      def scroll_y(event)
+        return nil unless event["type"] == INCREMENTAL
+
+        data = event["data"]
+        return nil unless data.is_a?(Hash) && data["source"] == SOURCE_SCROLL
+
+        id = data["id"]
+        return nil unless id.nil? || id == 1
+
+        y = data["y"]
+        y.is_a?(Numeric) ? y : nil
+      end
+
+      def viewport_height(segment)
+        height = meta_event(segment)&.dig("data", "height")
+        (height.is_a?(Numeric) && height > 0) ? height : nil
+      end
+
+      def meta_event(events)
+        events.find { |event| event["type"] == META && event["data"].is_a?(Hash) }
+      end
+
+      def summarize_samples(samples)
+        pixels = samples.map { |sample| sample[:max_y] }
+        page_height = samples.filter_map { |sample| viewport_bottom(sample) }.max
+        pcts = samples.filter_map { |sample| depth_pct(sample, page_height) }
+
+        {
+          session_count: samples.size,
+          avg_depth_px: mean(pixels),
+          avg_depth_pct: pcts.empty? ? nil : mean(pcts),
+          page_height_px: page_height,
+          fold_lines: fold_lines(pcts),
+          distribution: distribution(samples, page_height)
+        }
+      end
+
+      def viewport_bottom(sample)
+        height = sample[:viewport_height]
+        height ? sample[:max_y] + height : nil
+      end
+
+      def depth_pct(sample, page_height)
+        bottom = viewport_bottom(sample)
+        return nil unless bottom && page_height
+
+        [bottom.to_f / page_height * 100, 100.0].min
+      end
+
+      def fold_lines(pcts)
+        return {p50: nil, p75: nil, p90: nil} if pcts.empty?
+
+        sorted = pcts.sort
+        {p50: percentile(sorted, 50), p75: percentile(sorted, 75), p90: percentile(sorted, 90)}
+      end
+
+      # Viewport-less samples (no percentage derivable) fall back to pixels
+      # relative to the deepest sample so they still land in a bin.
+      def distribution(samples, page_height)
+        bins = DISTRIBUTION_BINS.to_h { |label| [label, 0] }
+        deepest_px = samples.map { |sample| sample[:max_y] }.max
+
+        samples.each do |sample|
+          pct = depth_pct(sample, page_height) || (sample[:max_y].to_f / deepest_px * 100)
+          bins[bin_for(pct)] += 1
+        end
+        bins
+      end
+
+      def bin_for(pct)
+        index = (pct / 25.0).ceil.clamp(1, 4) - 1
+        DISTRIBUTION_BINS[index]
+      end
+    end
+  end
+end

data/lib/sentiero/analytics/collectors/vitals_collector.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require_relative "../events"
+require_relative "../stats"
+
+module Sentiero
+  module Analytics
+    # Per-segment web-vitals accumulator. The recorder emits one "__perf" custom
+    # event per metric report, with data.payload {metric, value, rating}.
+    #
+    # Within a single segment, multiple reports for the same metric collapse to
+    # the LAST (the web-vitals library re-reports as the page evolves; only the
+    # final report is authoritative). Samples accumulate across all segments.
+    #
+    # #summarize's worst carries :value too; PageReportAnalyzer strips it afterward.
+    class VitalsCollector
+      include Events
+      include Stats
+
+      # Recorder tag for a web-vitals report.
+      PERF_TAG = "__perf"
+
+      attr_reader :capped
+
+      # max_samples: nil unbounded; an Integer caps each metric's values, flipping #capped.
+      def initialize(max_samples: nil)
+        @max_samples = max_samples
+        @metrics = {} # metric => {values:, ratings:, worst:}
+        @capped = false
+      end
+
+      # session_id/window_id/anchor attribute the worst (highest-value) sample.
+      # anchor is the window's first-event timestamp; offset_ms is relative to it
+      # so replay deep-links target the window start, not the segment.
+      def collect(segment, session_id:, window_id:, anchor:)
+        samples = {}
+        segment.each do |event|
+          sample = parse_sample(event)
+          samples[sample[:metric]] = sample if sample
+        end
+
+        samples.each_value do |sample|
+          entry = @metrics[sample[:metric]] ||= {values: [], ratings: Hash.new(0), worst: nil}
+          if @max_samples && entry[:values].size >= @max_samples
+            @capped = true
+            next
+          end
+
+          entry[:values] << sample[:value]
+
+          rating = sample[:rating]
+          entry[:ratings][rating] += 1 if rating.is_a?(String) && !rating.empty?
+
+          if entry[:worst].nil? || sample[:value] > entry[:worst][:value]
+            entry[:worst] = {
+              session_id: session_id,
+              window_id: window_id,
+              offset_ms: offset_ms(anchor, sample[:timestamp]),
+              value: sample[:value]
+            }
+          end
+        end
+      end
+
+      def summarize
+        summarized = @metrics.transform_values { |entry| summarize_metric(entry) }
+        {
+          sample_count: summarized.values.sum { |m| m[:samples] },
+          metrics: summarized
+        }
+      end
+
+      private
+
+      def parse_sample(event)
+        return nil unless event["type"] == CUSTOM
+
+        data = event["data"]
+        return nil unless data.is_a?(Hash) && data["tag"] == PERF_TAG
+
+        payload = data["payload"]
+        return nil unless payload.is_a?(Hash)
+
+        metric = payload["metric"]
+        value = payload["value"]
+        return nil unless metric.is_a?(String) && !metric.empty? && value.is_a?(Numeric)
+
+        {metric: metric, value: value, rating: payload["rating"], timestamp: event["timestamp"]}
+      end
+
+      def summarize_metric(entry)
+        sorted = entry[:values].sort
+        {
+          samples: sorted.size,
+          p50: percentile(sorted, 50),
+          p75: percentile(sorted, 75),
+          p90: percentile(sorted, 90),
+          ratings: entry[:ratings],
+          worst: entry[:worst]
+        }
+      end
+    end
+  end
+end

data/lib/sentiero/analytics/conversion_analyzer.rb

@@ -0,0 +1,247 @@
+# frozen_string_literal: true
+
+require "uri"
+
+require_relative "analyzer"
+require_relative "funnel_analyzer"
+
+module Sentiero
+  module Analytics
+    # Conversion rate by acquisition dimension (entry page, referrer host, UTM)
+    # for one custom-event tag. A session counts as converting at most once,
+    # regardless of how many times/windows the tag fired.
+    class ConversionAnalyzer < Analyzer
+      TOP_ROWS = 15
+
+      # Below this many sessions a rate is too thin; rows flagged low_volume.
+      MIN_SESSIONS_FOR_RATE = 5
+
+      # A new key past the cap is dropped and sets was_truncated.
+      MAX_DIMENSION_KEYS = 200
+
+      DIRECT = "(direct / none)"
+
+      # Matched case-insensitively.
+      UTM_PARAMS = %w[utm_source utm_medium utm_campaign].freeze
+
+      # No tag selected → empty facets, but tag vocabulary is still collected.
+      def analyze(tag = nil, limit: nil, since: nil, until_time: nil)
+        selected = FunnelAnalyzer.usable_steps([tag].compact).first
+
+        tags = {}
+        sessions = {}
+        @truncated = false
+
+        _scanned, hit_cap = scan_sessions(limit: limit, since: since, until_time: until_time) do |summary, window_id, events|
+          session_id = summary[:session_id]
+          state = sessions[session_id] ||= new_state(summary, window_id)
+
+          update_entry_candidate(state, events)
+          collect_vocabulary(tags, events)
+          record_conversion(state, selected, window_id, events) if selected
+        end
+
+        facets = selected ? build_facets(sessions) : empty_facets
+
+        {
+          tags: tags.keys.sort,
+          selected_tag: selected,
+          entry_pages: facets[:entry_pages],
+          referrers: facets[:referrers],
+          utm: facets[:utm],
+          was_truncated: @truncated || hit_cap
+        }
+      end
+
+      private
+
+      # entry_url precedence: an explicit entry_url is authoritative (anchor
+      # -Infinity so no later Meta can displace it), else the first Meta href wins.
+      def new_state(summary, first_window)
+        metadata = summary[:metadata] || {}
+        entry_url = metadata["entry_url"]
+        {
+          session_id: summary[:session_id],
+          entry_url: entry_url,
+          entry_anchor: entry_url ? -Float::INFINITY : nil,
+          referrer: metadata["entry_referrer"] || metadata["referrer"],
+          converted: false,
+          convert_window: nil,
+          convert_offset: nil,
+          convert_anchor: nil,
+          first_window: first_window
+        }
+      end
+
+      def collect_vocabulary(tags, events)
+        events.each do |event|
+          next unless event.is_a?(Hash) && event["type"] == CUSTOM
+          data = event["data"]
+          next unless data.is_a?(Hash)
+          tag = data["tag"]
+          next if FunnelAnalyzer.internal_tag?(tag)
+
+          next if tags.key?(tag)
+          if tags.size >= FunnelAnalyzer::MAX_TAGS
+            @truncated = true
+            next
+          end
+          tags[tag] = true
+        end
+      end
+
+      # Keeps the earliest conversion across windows so example coordinates are
+      # deterministic (earlier window start, or same start with earlier offset).
+      def record_conversion(state, tag, window_id, events)
+        anchor = events.first&.fetch("timestamp", nil)
+        match = events.find do |event|
+          event.is_a?(Hash) && event["type"] == CUSTOM &&
+            event["data"].is_a?(Hash) && event["data"]["tag"] == tag &&
+            event["timestamp"].is_a?(Numeric)
+        end
+        return unless match
+
+        offset = offset_ms(anchor, match["timestamp"])
+        return if state[:converted] && !earlier_match?(anchor, offset, state)
+
+        state[:converted] = true
+        state[:convert_window] = window_id
+        state[:convert_offset] = offset
+        state[:convert_anchor] = anchor
+      end
+
+      def earlier_match?(anchor, offset, state)
+        cur_anchor = state[:convert_anchor]
+        if anchor.is_a?(Numeric) && cur_anchor.is_a?(Numeric)
+          return true if anchor < cur_anchor
+          return false if anchor > cur_anchor
+          return offset < state[:convert_offset]
+        end
+        false
+      end
+
+      def empty_facets
+        {entry_pages: [], referrers: [], utm: {source: [], medium: [], campaign: []}}
+      end
+
+      # Runs after the scan, so every window of every session has been seen.
+      def build_facets(sessions)
+        entry_pages = new_facet
+        referrers = new_facet
+        utm = {source: new_facet, medium: new_facet, campaign: new_facet}
+
+        sessions.each_value do |state|
+          entry_url = state[:entry_url]
+          entry_key = normalize_entry(entry_url)
+          # No resolvable entry page => no acquisition data; contribute to no facet.
+          next unless entry_key
+
+          fold(entry_pages, entry_key, state)
+          fold(referrers, referrer_key(state[:referrer], entry_url), state)
+          fold_utm(utm, entry_url, state)
+        end
+
+        {
+          entry_pages: rows_for(entry_pages),
+          referrers: rows_for(referrers),
+          utm: {
+            source: rows_for(utm[:source]),
+            medium: rows_for(utm[:medium]),
+            campaign: rows_for(utm[:campaign])
+          }
+        }
+      end
+
+      def new_facet
+        {sessions: Hash.new(0), conversions: Hash.new(0), converting: {}, non_converting: {}}
+      end
+
+      def fold(facet, key, state)
+        return if key.nil?
+
+        if !facet[:sessions].key?(key) && facet[:sessions].size >= MAX_DIMENSION_KEYS
+          @truncated = true
+          return
+        end
+
+        facet[:sessions][key] += 1
+        if state[:converted]
+          facet[:conversions][key] += 1
+          facet[:converting][key] ||= {
+            session_id: state[:session_id],
+            window_id: state[:convert_window],
+            offset_ms: state[:convert_offset]
+          }
+        else
+          facet[:non_converting][key] ||= {
+            session_id: state[:session_id],
+            window_id: state[:first_window],
+            offset_ms: 0
+          }
+        end
+      end
+
+      def fold_utm(utm, entry_url, state)
+        params = utm_params(entry_url)
+        fold(utm[:source], params["utm_source"], state)
+        fold(utm[:medium], params["utm_medium"], state)
+        fold(utm[:campaign], params["utm_campaign"], state)
+      end
+
+      def normalize_entry(url)
+        return nil unless url.is_a?(String) && !url.empty?
+
+        uri = URI.parse(url)
+        return nil unless uri.scheme && uri.host
+
+        port = (uri.port && uri.port != uri.default_port) ? ":#{uri.port}" : ""
+        "#{uri.scheme}://#{uri.host}#{port}#{uri.path}"
+      rescue URI::InvalidURIError
+        nil
+      end
+
+      # Same-origin referrers are from within the site, not acquisition, so dropped.
+      def referrer_key(referrer, entry_url)
+        return nil if same_origin?(referrer, entry_url)
+        return DIRECT unless referrer.is_a?(String) && !referrer.empty?
+
+        host = URI.parse(referrer).host
+        (host && !host.empty?) ? host : DIRECT
+      rescue URI::InvalidURIError
+        DIRECT
+      end
+
+      def utm_params(url)
+        out = {}
+        return out unless url.is_a?(String) && url.include?("?")
+
+        query = url.split("?", 2)[1].split("#", 2)[0]
+        URI.decode_www_form(query).each do |key, value|
+          name = key.to_s.downcase
+          next unless UTM_PARAMS.include?(name)
+          next if out.key?(name)
+          stripped = value.to_s.strip
+          out[name] = stripped unless stripped.empty?
+        end
+        out
+      rescue ArgumentError
+        out
+      end
+
+      def rows_for(facet)
+        top_counts(facet[:sessions], limit: TOP_ROWS).map do |key, sessions|
+          conversions = facet[:conversions][key]
+          {
+            key: key,
+            sessions: sessions,
+            conversions: conversions,
+            conversion_rate: sessions.zero? ? nil : (conversions.to_f / sessions * 100).round(1),
+            low_volume: sessions < MIN_SESSIONS_FOR_RATE,
+            converting_example: facet[:converting][key],
+            non_converting_example: facet[:non_converting][key]
+          }
+        end
+      end
+    end
+  end
+end