sentiero 1.0.0.alpha1 → 1.0.0.beta3

data/lib/sentiero/fingerprint/config.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Sentiero
+  module Fingerprint
+    # Registry of per-platform frame normalizers, reached as
+    # `Sentiero.configuration.fingerprint`. `ErrorsApp` resolves an incoming
+    # occurrence's optional "platform" tag against this registry to pick the
+    # normalizer passed to Fingerprint.compute.
+    class Config
+      attr_accessor :default_platform
+
+      def initialize
+        @normalizers = {}
+        @default_platform = "ruby"
+        register("ruby", RUBY_NORMALIZER)
+        register("crystal", CRYSTAL_NORMALIZER)
+        register("generic", GENERIC_NORMALIZER)
+      end
+
+      # Registering an existing name overrides it, so operators may replace a
+      # built-in (e.g. swap out "ruby") as well as add new platforms.
+      def register(name, callable)
+        @normalizers[name.to_s] = callable
+      end
+
+      # Three-tier resolution (see design doc): a reporter that never learned
+      # about the "platform" field must keep grouping exactly as before
+      # (absent/blank -> default_platform's normalizer, tier 1), whereas a
+      # reporter that declares a platform we don't recognize should not have a
+      # foreign grammar mis-applied to it (unregistered -> generic, tier 3).
+      def resolve(platform)
+        if platform.nil? || platform.to_s.strip.empty?
+          @normalizers.fetch(default_platform.to_s, RUBY_NORMALIZER)
+        else
+          @normalizers.fetch(platform.to_s, GENERIC_NORMALIZER)
+        end
+      end
+    end
+  end
+end

data/lib/sentiero/fingerprint.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "digest"
+require_relative "fingerprint/config"
 
 module Sentiero
   # Computes the grouping fingerprint for an exception occurrence. The
@@ -12,23 +13,56 @@ module Sentiero
     MAX_FRAMES = 5
     MAX_FRAME_LENGTH = 1000
 
+    # `path:LINE:in 'method'` (colon-glued) or bare `path:LINE`. Verbatim
+    # pre-refactor logic, kept byte-identical so upgrading does not fracture
+    # existing problem groups — this is the tier-1 (no platform tag) default.
+    RUBY_NORMALIZER = ->(frame) {
+      frame
+        .gsub(/0x[0-9a-fA-F]+/, "0xHEX") # memory addresses
+        .gsub(/:[0-9]+(?=:in )/, ":N") # `path:LINE:in 'method'`
+        .gsub(/:[0-9]+\z/, ":N") # `path:LINE` (top-level frame, no method)
+    }
+
+    # `path.cr:LINE in 'method'` or `path.cr:LINE:COL in 'method'` — Crystal
+    # separates the line marker from `in` with a space, not a colon, and may
+    # carry an optional `:column`, so the Ruby regex is a no-op on it.
+    CRYSTAL_NORMALIZER = ->(frame) {
+      frame
+        .gsub(/0x[0-9a-fA-F]+/, "0xHEX")
+        .gsub(/:[0-9]+(?::[0-9]+)?(?= in )/, ":N")
+    }
+
+    # Grammar-agnostic fallback: strips a trailing `:LINE` or `:LINE:COL` only,
+    # with no `in`-token assumption. Coarser than a language-specific
+    # normalizer (it can't see line noise that isn't at the end of the frame)
+    # but never mis-applies a foreign grammar.
+    GENERIC_NORMALIZER = ->(frame) {
+      frame
+        .gsub(/0x[0-9a-fA-F]+/, "0xHEX")
+        .gsub(/:[0-9]+(?::[0-9]+)?\z/, ":N")
+    }
+
     module_function
 
-    def compute(exception_class:, backtrace:, project:)
-      frames = Array(backtrace).first(MAX_FRAMES).map { |frame| normalize_frame(frame.to_s) }
+    def compute(exception_class:, backtrace:, project:, normalizer: RUBY_NORMALIZER)
+      frames = Array(backtrace).first(MAX_FRAMES).map { |frame| safe_normalize(normalizer, frame.to_s) }
       input = "#{project}\n#{exception_class}\n#{frames.join("\n")}"
       Digest::SHA256.hexdigest(input)[0, 40]
     end
 
-    # Strips per-occurrence noise (memory addresses, line numbers). Digits inside
-    # identifiers (e.g. `step_1`, `V2::Api`) are preserved so distinct methods do
-    # not collapse into one group.
-    def normalize_frame(frame)
-      frame = frame[0, MAX_FRAME_LENGTH].strip
-      frame
-        .gsub(/0x[0-9a-fA-F]+/, "0xHEX") # memory addresses
-        .gsub(/:[0-9]+(?=:in )/, ":N") # `path:LINE:in 'method'`
-        .gsub(/:[0-9]+\z/, ":N") # `path:LINE` (top-level frame, no method)
+    # Internal helper, not part of the public API. Wraps a (possibly
+    # user-registered) normalizer so a broken or pathological callable can
+    # never break fingerprinting: the frame is capped before and after
+    # normalization, and any exception falls back to the raw capped frame.
+    # SystemStackError and NoMemoryError are non-StandardError exceptions a
+    # pathological normalizer can realistically raise (e.g. unbounded
+    # recursion); a bare `rescue Exception` is deliberately avoided so
+    # Interrupt/SystemExit still propagate.
+    def safe_normalize(normalizer, frame)
+      capped = frame[0, MAX_FRAME_LENGTH].strip
+      normalizer.call(capped).to_s[0, MAX_FRAME_LENGTH]
+    rescue StandardError, SystemStackError, NoMemoryError
+      capped
     end
   end
 end

data/lib/sentiero/geo.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Sentiero
+  # Deliberately coarse — country/city/region/timezone, no coordinates, no IP:
+  # resolution happens in-request and only the result is stored, so
+  # config.anonymize_ip is unaffected.
+  module Geo
+    MAX_VALUE_LENGTH = 256
+    PROC_WARNING_LOCK = Mutex.new
+    @proc_warning_emitted = false
+
+    # CF-IPCountry ships with Cloudflare IP geolocation; the other headers need
+    # the "Add visitor location headers" managed transform, so country-only is the common case.
+    CLOUDFLARE_HEADERS = {
+      "geo_country" => "HTTP_CF_IPCOUNTRY",
+      "geo_city" => "HTTP_CF_IPCITY",
+      "geo_region" => "HTTP_CF_REGION",
+      "geo_timezone" => "HTTP_CF_TIMEZONE"
+    }.freeze
+
+    # Cloudflare placeholder codes: XX = unknown, T1 = Tor exit.
+    CLOUDFLARE_UNKNOWN = %w[XX T1].freeze
+
+    PROC_KEY_MAP = {
+      "country" => "geo_country",
+      "city" => "geo_city",
+      "region" => "geo_region",
+      "timezone" => "geo_timezone"
+    }.freeze
+
+    TARGET_KEYS = PROC_KEY_MAP.values.freeze
+
+    module_function
+
+    def resolve(env, source, client_metadata = {})
+      case source
+      when nil then {}
+      when :cloudflare then from_cloudflare(env)
+      else from_proc(env, source, client_metadata)
+      end
+    end
+
+    def from_cloudflare(env)
+      geo = CLOUDFLARE_HEADERS.each_with_object({}) do |(key, header), acc|
+        value = clean(env[header])
+        acc[key] = value if value
+      end
+      geo.delete("geo_country") if CLOUDFLARE_UNKNOWN.include?(geo["geo_country"])
+      geo
+    end
+
+    # A broken geo hook must never break ingest: rescue everything, warn once
+    # per process, resolve empty. A two-arg resolver opts into the client metadata.
+    def from_proc(env, source, client_metadata = {})
+      raw = (source.arity == 1) ? source.call(env) : source.call(env, client_metadata)
+      return {} unless raw.is_a?(Hash)
+
+      raw.each_with_object({}) do |(key, value), acc|
+        mapped = PROC_KEY_MAP[key.to_s]
+        value = clean(value)
+        acc[mapped] = value if mapped && value
+      end
+    rescue => e
+      PROC_WARNING_LOCK.synchronize do
+        return {} if @proc_warning_emitted
+        @proc_warning_emitted = true
+      end
+
+      warn "[Sentiero] geo_source raised #{e.class}: #{e.message}; skipping geo capture"
+      {}
+    end
+
+    def reset_proc_warning!
+      @proc_warning_emitted = false
+    end
+
+    def clean(value)
+      return nil unless value.is_a?(String)
+
+      stripped = value.strip
+      return nil if stripped.empty?
+
+      stripped[0, MAX_VALUE_LENGTH]
+    end
+  end
+end

data/lib/sentiero/reporter.rb

@@ -145,7 +145,8 @@ module Sentiero
           "message" => Redaction.redact_text(exception.message.to_s, config),
           "backtrace" => Array(exception.backtrace).map { |frame| Redaction.redact_text(frame.to_s, config) },
           "context" => Redaction.deep_redact_strings(scrubber.scrub(report_ctx.metadata), config),
-          "timestamp" => Time.now.to_f
+          "timestamp" => Time.now.to_f,
+          "platform" => "ruby"
         }
         payload["session_id"] = report_ctx.session_id if report_ctx.session_id
         payload["window_id"] = report_ctx.window_id if report_ctx.window_id

data/lib/sentiero/store/session_store.rb

@@ -12,7 +12,12 @@ module Sentiero
         raise NoMethodError, "#{self.class}#save_events not implemented"
       end
 
-      def list_sessions(limit:, offset: 0, since: nil, until_time: nil, sort_by: nil, search: nil)
+      # min_duration_ms filters on the event-timestamp span (last_event_at -
+      # first_event_at, milliseconds); sessions without event timestamps fail
+      # an active duration filter. Both thresholds are inclusive and apply
+      # before limit/offset.
+      def list_sessions(limit:, offset: 0, since: nil, until_time: nil, sort_by: nil, search: nil,
+        min_duration_ms: nil, min_events: nil)
         raise NoMethodError, "#{self.class}#list_sessions not implemented"
       end
 
@@ -40,8 +45,16 @@ module Sentiero
       # Yields [session_summary_hash, window_id, events_array] per window, newest
       # sessions first, capped at `limit`. Built from list_sessions/get_session/
       # get_events so every backend gets it free; stores may override.
-      def each_session_events(limit: nil, since: nil, until_time: nil)
-        return enum_for(:each_session_events, limit: limit, since: since, until_time: until_time) unless block_given?
+      #
+      # `types:` (integer rrweb event types) narrows only the yielded events
+      # arrays. Summaries and the yielded window set always describe the whole
+      # session, and carry :window_starts ({window_id => first event
+      # timestamp}) so replay offsets stay anchored when a window's first
+      # event is filtered out.
+      def each_session_events(limit: nil, since: nil, until_time: nil, types: nil)
+        unless block_given?
+          return enum_for(:each_session_events, limit: limit, since: since, until_time: until_time, types: types)
+        end
 
         cap = limit || limits.analytics_max_scan_sessions
         sessions = list_sessions(limit: cap, since: since, until_time: until_time)
@@ -51,14 +64,49 @@ module Sentiero
           next unless session
 
           windows = session[:windows] || []
+          summary = summary.merge(
+            window_starts: windows.to_h { |window| [window[:window_id], window[:first_event_at]] }
+          )
           windows.each do |window|
             window_id = window[:window_id]
             events = get_events(WindowRef.new(summary[:session_id], window_id))
+            events = events.select { |event| types.include?(event["type"]) } if types
             yield summary, window_id, events
           end
         end
       end
 
+      # {integer_type => count}; non-integer client "type" values tally under
+      # nil. SQLite overrides with a payload-free aggregate.
+      def event_type_counts(limit: nil, since: nil, until_time: nil)
+        counts = {}
+        each_session_events(limit: limit, since: since, until_time: until_time) do |_summary, _window_id, events|
+          events.each do |event|
+            type = event["type"]
+            key = type.is_a?(Integer) ? type : nil
+            counts[key] = (counts[key] || 0) + 1
+          end
+        end
+        counts
+      end
+
+      # {"YYYY-MM-DD" (UTC day of the ms timestamp) => count}; events without
+      # a numeric timestamp are skipped. SQLite overrides with a payload-free
+      # aggregate.
+      def event_counts_by_day(limit: nil, since: nil, until_time: nil)
+        counts = {}
+        each_session_events(limit: limit, since: since, until_time: until_time) do |_summary, _window_id, events|
+          events.each do |event|
+            timestamp = event["timestamp"]
+            next unless timestamp.is_a?(Numeric)
+
+            day = Time.at(timestamp / 1000.0).utc.to_date.to_s
+            counts[day] = (counts[day] || 0) + 1
+          end
+        end
+        counts
+      end
+
       # Deletes every session whose updated_at is older than `seconds` ago,
       # returning the count. Built from list_sessions + delete_session so every
       # backend gets it free; stores may override with a direct query.
@@ -106,6 +154,17 @@ module Sentiero
         entry
       end
 
+      def session_meets_thresholds?(summary, min_duration_ms, min_events)
+        return false if min_events && summary[:event_count] < min_events
+
+        if min_duration_ms
+          first = summary[:first_event_at]
+          last = summary[:last_event_at]
+          return false unless first && last && (last - first) >= min_duration_ms
+        end
+        true
+      end
+
       # True when the search term (case-insensitive) appears in the session_id
       # or any metadata value.
       def session_matches_search?(summary, search)

data/lib/sentiero/store.rb

@@ -28,6 +28,13 @@ module Sentiero
       @limits ||= Limits.new
     end
 
+    # True when event_type_counts / event_counts_by_day avoid reading event
+    # payloads (native aggregates). Via the scan-based fallbacks each figure
+    # would re-scan the store, so StatsAggregator only calls them when true.
+    def supports_event_aggregates?
+      false
+    end
+
     private
 
     def validate_id!(id)

data/lib/sentiero/stores/file.rb

@@ -60,7 +60,8 @@ module Sentiero
         nil
       end
 
-      def list_sessions(limit:, offset: 0, since: nil, until_time: nil, sort_by: nil, search: nil)
+      def list_sessions(limit:, offset: 0, since: nil, until_time: nil, sort_by: nil, search: nil,
+        min_duration_ms: nil, min_events: nil)
         session_ids = list_session_ids
         return [] if session_ids.empty?
 
@@ -77,6 +78,9 @@ module Sentiero
         if search && !search.empty?
           summaries = summaries.select { |summary| session_matches_search?(summary, search) }
         end
+        if min_duration_ms || min_events
+          summaries = summaries.select { |summary| session_meets_thresholds?(summary, min_duration_ms, min_events) }
+        end
 
         case sort_by
         when "created_at"

data/lib/sentiero/stores/memory.rb

@@ -64,9 +64,11 @@ module Sentiero
         nil
       end
 
-      def list_sessions(limit:, offset: 0, since: nil, until_time: nil, sort_by: nil, search: nil)
+      def list_sessions(limit:, offset: 0, since: nil, until_time: nil, sort_by: nil, search: nil,
+        min_duration_ms: nil, min_events: nil)
         pairs = @sessions.each_pair.to_a
-        pairs = filter_sessions(pairs, since: since, until_time: until_time, search: search)
+        pairs = filter_sessions(pairs, since: since, until_time: until_time, search: search,
+          min_duration_ms: min_duration_ms, min_events: min_events)
         pairs = sort_sessions(pairs, sort_by)
         page = pairs.slice(offset, limit) || []
         page.map { |sid, entry| session_summary(sid, entry) }
@@ -274,7 +276,7 @@ module Sentiero
 
       private
 
-      def filter_sessions(pairs, since:, until_time:, search:)
+      def filter_sessions(pairs, since:, until_time:, search:, min_duration_ms: nil, min_events: nil)
         if since
           since_f = since.to_f
           pairs = pairs.select { |_sid, entry| entry.meta.updated_at >= since_f }
@@ -290,6 +292,16 @@ module Sentiero
               entry.meta.session_metadata&.values&.any? { |value| value.to_s.downcase.include?(search_down) }
           }
         end
+        if min_events
+          pairs = pairs.select { |_sid, entry| entry.windows.values.sum(&:size) >= min_events }
+        end
+        if min_duration_ms
+          pairs = pairs.select { |_sid, entry|
+            first = entry.meta.first_event_at
+            last = entry.meta.last_event_at
+            first && last && (last - first) >= min_duration_ms
+          }
+        end
         pairs
       end
 

data/lib/sentiero/stores/redis.rb

@@ -76,8 +76,10 @@ module Sentiero
 
       # Batched scan: three pipelined round-trips total instead of the base's
       # get_session + get_events per window.
-      def each_session_events(limit: nil, since: nil, until_time: nil)
-        return enum_for(:each_session_events, limit: limit, since: since, until_time: until_time) unless block_given?
+      def each_session_events(limit: nil, since: nil, until_time: nil, types: nil)
+        unless block_given?
+          return enum_for(:each_session_events, limit: limit, since: since, until_time: until_time, types: types)
+        end
 
         cap = limit || limits.analytics_max_scan_sessions
         min = since ? since.to_f.to_s : "-inf"
@@ -114,7 +116,13 @@ module Sentiero
 
           windows = window_ids.to_h { |wid| [wid, parse_events(event_futures[[sid, wid]].value)] }
           summary = scan_summary(sid, meta, window_ids, windows.values.sum(&:size))
-          windows.each { |wid, events| yield summary, wid, events }
+          summary = summary.merge(
+            window_starts: windows.transform_values { |events| events.first&.fetch("timestamp", nil) }
+          )
+          windows.each do |wid, events|
+            events = events.select { |event| types.include?(event["type"]) } if types
+            yield summary, wid, events
+          end
         end
       end
 
@@ -122,11 +130,12 @@ module Sentiero
       # ZREVRANGEBYSCORE with LIMIT pages the score-ordered sessions zset
       # directly. created_at/event_count sort or search need every matching
       # session's summary first, so those go through the general path below.
-      def list_sessions(limit:, offset: 0, since: nil, until_time: nil, sort_by: nil, search: nil)
+      def list_sessions(limit:, offset: 0, since: nil, until_time: nil, sort_by: nil, search: nil,
+        min_duration_ms: nil, min_events: nil)
         min = since ? since.to_f.to_s : "-inf"
         max = until_time ? until_time.to_f.to_s : "+inf"
 
-        if default_sort?(sort_by) && (search.nil? || search.empty?)
+        if default_sort?(sort_by) && (search.nil? || search.empty?) && min_duration_ms.nil? && min_events.nil?
           session_ids = @redis.zrevrangebyscore(@keys.sessions, max, min, limit: [offset, limit])
           return build_session_summaries(session_ids)
         end
@@ -137,6 +146,9 @@ module Sentiero
         summaries = build_session_summaries(session_ids)
 
         summaries.select! { |summary| session_matches_search?(summary, search) } if search && !search.empty?
+        if min_duration_ms || min_events
+          summaries.select! { |summary| session_meets_thresholds?(summary, min_duration_ms, min_events) }
+        end
 
         case sort_by
         when "created_at"

data/lib/sentiero/stores/sqlite/payload_codec.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "zlib"
+
+module Sentiero
+  module Stores
+    class SQLite
+      # zlib compression for events.data payloads. FullSnapshots dominate the
+      # table's bytes (~93% in production samples) and deflate ~6-7x, so
+      # payloads at or above the threshold are stored as compressed BLOBs;
+      # smaller rows stay plain JSON TEXT (near-zero win, and skipping them
+      # keeps ~85% of rows out of the deflate path). Reads sniff the storage
+      # format from the payload itself: a zlib stream starts with 0x78 and a
+      # valid header checksum, which no JSON text can.
+      module PayloadCodec
+        COMPRESSION_THRESHOLD = 1024
+        # CMF byte for deflate with a 32K window — the only one zlib emits.
+        ZLIB_FIRST_BYTE = 0x78
+
+        module_function
+
+        def encode(json)
+          return json if json.bytesize < COMPRESSION_THRESHOLD
+
+          ::SQLite3::Blob.new(Zlib::Deflate.deflate(json))
+        end
+
+        def decode(data)
+          return data unless compressed?(data)
+
+          Zlib::Inflate.inflate(data).force_encoding(Encoding::UTF_8)
+        end
+
+        def compressed?(data)
+          data.getbyte(0) == ZLIB_FIRST_BYTE &&
+            (flg = data.getbyte(1)) &&
+            (((ZLIB_FIRST_BYTE << 8) | flg) % 31).zero?
+        end
+      end
+    end
+  end
+end

data/lib/sentiero/stores/sqlite/schema.rb

@@ -3,8 +3,21 @@
 module Sentiero
   module Stores
     class SQLite
-      # Table/index definitions for the store's single SQLite3 connection.
+      # Table/index definitions for the store's SQLite connections.
       module Schema
+        # The events.type column (denormalized rrweb event type) sits BEFORE
+        # the data blob on purpose: SQLite reads record columns left to right,
+        # so a typed scan can check `type` without ever touching the (often
+        # multi-page overflow) data payload of non-matching rows.
+        EVENTS_COLUMNS = <<~SQL
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          session_id TEXT NOT NULL,
+          window_id TEXT NOT NULL,
+          timestamp REAL,
+          type INTEGER,
+          data TEXT NOT NULL
+        SQL
+
         def self.create(db)
           db.execute_batch(<<~SQL)
             CREATE TABLE IF NOT EXISTS sessions (
@@ -21,11 +34,7 @@ module Sentiero
             CREATE INDEX IF NOT EXISTS idx_sessions_updated_at ON sessions(updated_at);
 
             CREATE TABLE IF NOT EXISTS events (
-              id INTEGER PRIMARY KEY AUTOINCREMENT,
-              session_id TEXT NOT NULL,
-              window_id TEXT NOT NULL,
-              timestamp REAL,
-              data TEXT NOT NULL
+              #{EVENTS_COLUMNS}
             );
 
             CREATE INDEX IF NOT EXISTS idx_events_session_window_ts ON events(session_id, window_id, timestamp);
@@ -72,6 +81,63 @@ module Sentiero
             CREATE INDEX IF NOT EXISTS idx_server_events_project ON server_events(project);
             CREATE INDEX IF NOT EXISTS idx_server_events_session ON server_events(session_id);
           SQL
+
+          migrate_events_type!(db)
+          migrate_compress_payloads!(db)
+        end
+
+        # Databases created before the events.type column existed get a
+        # one-time table rebuild (not ALTER ADD COLUMN — that would append
+        # type AFTER the data blob, defeating the column-order optimization
+        # above). Backfills type from each row's JSON payload.
+        def self.migrate_events_type!(db)
+          columns = db.execute("PRAGMA table_info(events)").map { |row| row["name"] }
+          return if columns.include?("type")
+
+          db.transaction(:immediate) do
+            db.execute_batch(<<~SQL)
+              CREATE TABLE events_migrated (
+                #{EVENTS_COLUMNS}
+              );
+              INSERT INTO events_migrated (id, session_id, window_id, timestamp, type, data)
+                SELECT id, session_id, window_id, timestamp,
+                       CASE WHEN json_type(data, '$.type') = 'integer'
+                            THEN json_extract(data, '$.type') END,
+                       data
+                FROM events;
+              DROP TABLE events;
+              ALTER TABLE events_migrated RENAME TO events;
+              CREATE INDEX idx_events_session_window_ts ON events(session_id, window_id, timestamp);
+            SQL
+          end
+        end
+
+        # user_version 1 = every large payload is zlib-compressed (PayloadCodec).
+        SCHEMA_VERSION = 1
+
+        # One-time pass compressing pre-existing plain-TEXT payloads. Batched
+        # in short transactions so a live recorder's ingest writes interleave
+        # instead of timing out behind one giant lock. Guarded by
+        # PRAGMA user_version, so subsequent opens skip the table scan.
+        # Frees pages for reuse but does not shrink the file — VACUUM manually
+        # to reclaim disk.
+        def self.migrate_compress_payloads!(db)
+          return if db.get_first_value("PRAGMA user_version").to_i >= SCHEMA_VERSION
+
+          loop do
+            rows = db.execute(
+              "SELECT id, data FROM events WHERE typeof(data) = 'text' AND LENGTH(data) >= ? LIMIT 500",
+              [PayloadCodec::COMPRESSION_THRESHOLD]
+            )
+            break if rows.empty?
+
+            db.transaction(:immediate) do
+              rows.each do |row|
+                db.execute("UPDATE events SET data = ? WHERE id = ?", [PayloadCodec.encode(row["data"]), row["id"]])
+              end
+            end
+          end
+          db.execute("PRAGMA user_version = #{SCHEMA_VERSION}")
         end
       end
     end