sentiero 1.0.0.beta1 → 1.0.0.beta3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c8bc35b365449e9111ed5dcc093ea7fceaf60d739b47e1bf13df8f7cbc83cf55
-  data.tar.gz: 39c9a76c8acf2f22d44fcc3fa88d450ef1e36a368434ba5ae143c82bd2800b30
+  metadata.gz: ed5c945cbabf6b9ae1ab2b3e10474d06bc06bda78c1a4b381b69845791c18296
+  data.tar.gz: abb85bfebbdb1e8c48cba2eb844b6446d5bd82f17e7727c3b00347b5256de38e
 SHA512:
-  metadata.gz: 527083a0dc823bb03343f97b41f3f40e3c629ce201dbf234613c9d615068e8e481d612f0aa0ff8a5736e6cc487f1f6d049f0d704647b0fa37e7361d4cafa4003
-  data.tar.gz: c77ebaff3d74da9cdef241db684ccb6f0b5a141a10932fd49d3af131bd4434ddb8194ee78abab087a16893c2b01a4566f0f75f98f44e17679e5107bfb2a9b0dc
+  metadata.gz: c5ef1855f03fc35d2b719ee316979de2ae3331b2400bfb534a0406048fe31cd9b292e252fdda5a59f54f73a066c33aed17aecaf88c2e697a82d0269ddd525f84
+  data.tar.gz: c7fce88e343ca579a5add1656f6e12b997162a5c7dbe62630e21b00334e13e3bc88f51dcabc9878f8ad93712851b626f5475cd76bec870eb1f142a7089959021

data/lib/sentiero/analytics/error_discovery.rb

@@ -48,6 +48,7 @@ module Sentiero
 
         key = ErrorCollector.group_key(message)
         group = groups[key] ||= new_group(key, message, payload)
+        group[:stack] ||= stack_of(payload)
         group[:count] += 1
         group[:last_seen_at] = [group[:last_seen_at], timestamp].compact.max
         tally_facets(group, summary[:metadata] || {})
@@ -72,6 +73,7 @@ module Sentiero
           message: message,
           source: source_of(payload),
           line: line_of(payload),
+          stack: nil,
           count: 0,
           last_seen_at: nil,
           browsers: Hash.new(0),
@@ -105,6 +107,11 @@ module Sentiero
         line.is_a?(Integer) ? line : nil
       end
 
+      def stack_of(payload)
+        stack = payload["stack"]
+        (stack.is_a?(String) && !stack.empty?) ? stack : nil
+      end
+
       def sort_groups(groups, sort_by)
         case sort_by
         when "recency"

data/lib/sentiero/stores/sqlite/payload_codec.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "zlib"
+
+module Sentiero
+  module Stores
+    class SQLite
+      # zlib compression for events.data payloads. FullSnapshots dominate the
+      # table's bytes (~93% in production samples) and deflate ~6-7x, so
+      # payloads at or above the threshold are stored as compressed BLOBs;
+      # smaller rows stay plain JSON TEXT (near-zero win, and skipping them
+      # keeps ~85% of rows out of the deflate path). Reads sniff the storage
+      # format from the payload itself: a zlib stream starts with 0x78 and a
+      # valid header checksum, which no JSON text can.
+      module PayloadCodec
+        COMPRESSION_THRESHOLD = 1024
+        # CMF byte for deflate with a 32K window — the only one zlib emits.
+        ZLIB_FIRST_BYTE = 0x78
+
+        module_function
+
+        def encode(json)
+          return json if json.bytesize < COMPRESSION_THRESHOLD
+
+          ::SQLite3::Blob.new(Zlib::Deflate.deflate(json))
+        end
+
+        def decode(data)
+          return data unless compressed?(data)
+
+          Zlib::Inflate.inflate(data).force_encoding(Encoding::UTF_8)
+        end
+
+        def compressed?(data)
+          data.getbyte(0) == ZLIB_FIRST_BYTE &&
+            (flg = data.getbyte(1)) &&
+            (((ZLIB_FIRST_BYTE << 8) | flg) % 31).zero?
+        end
+      end
+    end
+  end
+end

data/lib/sentiero/stores/sqlite/schema.rb

@@ -83,6 +83,7 @@ module Sentiero
           SQL
 
           migrate_events_type!(db)
+          migrate_compress_payloads!(db)
         end
 
         # Databases created before the events.type column existed get a
@@ -110,6 +111,34 @@ module Sentiero
             SQL
           end
         end
+
+        # user_version 1 = every large payload is zlib-compressed (PayloadCodec).
+        SCHEMA_VERSION = 1
+
+        # One-time pass compressing pre-existing plain-TEXT payloads. Batched
+        # in short transactions so a live recorder's ingest writes interleave
+        # instead of timing out behind one giant lock. Guarded by
+        # PRAGMA user_version, so subsequent opens skip the table scan.
+        # Frees pages for reuse but does not shrink the file — VACUUM manually
+        # to reclaim disk.
+        def self.migrate_compress_payloads!(db)
+          return if db.get_first_value("PRAGMA user_version").to_i >= SCHEMA_VERSION
+
+          loop do
+            rows = db.execute(
+              "SELECT id, data FROM events WHERE typeof(data) = 'text' AND LENGTH(data) >= ? LIMIT 500",
+              [PayloadCodec::COMPRESSION_THRESHOLD]
+            )
+            break if rows.empty?
+
+            db.transaction(:immediate) do
+              rows.each do |row|
+                db.execute("UPDATE events SET data = ? WHERE id = ?", [PayloadCodec.encode(row["data"]), row["id"]])
+              end
+            end
+          end
+          db.execute("PRAGMA user_version = #{SCHEMA_VERSION}")
+        end
       end
     end
   end

data/lib/sentiero/stores/sqlite.rb

@@ -16,6 +16,7 @@ module Sentiero
     class SQLite < Store
       # Loaded after the class line above establishes SQLite < Store, so
       # schema.rb's own `class SQLite` reopen doesn't hit a superclass mismatch.
+      require_relative "sqlite/payload_codec"
       require_relative "sqlite/schema"
 
       # Single-file SQLite store for single-process production and dev.
@@ -101,7 +102,7 @@ module Sentiero
             events.each do |event|
               type = event["type"]
               stmt.execute(session_id, window_id, event["timestamp"]&.to_f,
-                type.is_a?(Integer) ? type : nil, JSON.generate(event))
+                type.is_a?(Integer) ? type : nil, PayloadCodec.encode(JSON.generate(event)))
             end
           ensure
             stmt.close
@@ -300,7 +301,7 @@ module Sentiero
           params << limit.to_i
         end
 
-        db.execute(sql, params).map { |event_row| JSON.parse(event_row["data"]) }
+        db.execute(sql, params).map { |event_row| JSON.parse(PayloadCodec.decode(event_row["data"])) }
       end
 
       def save_metadata(session_id, metadata)
@@ -570,7 +571,7 @@ module Sentiero
             "SELECT session_id, window_id, data FROM events WHERE session_id IN (#{placeholders})#{type_clause} ORDER BY timestamp ASC",
             chunk + type_params
           ).each do |row|
-            (grouped[row["session_id"]][row["window_id"]] ||= []) << JSON.parse(row["data"])
+            (grouped[row["session_id"]][row["window_id"]] ||= []) << JSON.parse(PayloadCodec.decode(row["data"]))
           end
         end
         grouped

data/lib/sentiero/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sentiero
-  VERSION = "1.0.0.beta1"
+  VERSION = "1.0.0.beta3"
 end

data/lib/sentiero/web/templates/client_error_show.html.erb

@@ -23,6 +23,11 @@
       <div class="text-xs text-gray-600">
         <%= view.group[:last_seen_at] ? Time.at(view.group[:last_seen_at].to_f / 1000).utc.strftime("%b %d, %Y %H:%M:%S UTC") : "N/A" %>
       </div>
+
+      <% if view.group[:stack] -%>
+        <div class="text-[10px] font-medium text-gray-400 uppercase tracking-wider pt-0.5">Stack</div>
+        <pre class="text-[10px] text-gray-500 bg-gray-50 rounded p-2 overflow-x-auto whitespace-pre-wrap font-mono" data-client-error-stack><%= view.h(view.group[:stack]) %></pre>
+      <% end -%>
     </div>
   </div>
 </div>
@@ -36,7 +41,7 @@
       <% view.facet_chips.each do |title, counts| -%>
         <% next if counts.empty? -%>
         <div class="flex items-center gap-2 mb-1.5 flex-wrap">
-          <span class="text-[10px] font-medium text-gray-400 uppercase tracking-wider w-16 shrink-0"><%= view.h(title) %></span>
+          <span class="text-[10px] font-medium text-gray-400 uppercase tracking-wider w-20 shrink-0"><%= view.h(title) %></span>
           <% counts.sort_by { |_value, count| -count }.first(8).each do |value, count| -%>
             <span class="badge badge-neutral"><%= view.h(value) %>&nbsp;<span class="tabular-nums">&times;<%= count %></span></span>
           <% end -%>

data/lib/sentiero/web/views/client_error_show_view.rb

@@ -20,7 +20,7 @@ module Sentiero
           [
             ["Browsers", group[:browsers] || {}],
             ["Devices", group[:devices] || {}],
-            ["Pages", group[:pages] || {}]
+            ["Entry pages", group[:pages] || {}]
           ]
         end
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sentiero
 version: !ruby/object:Gem::Version
-  version: 1.0.0.beta1
+  version: 1.0.0.beta3
 platform: ruby
 authors:
 - Stephen Ierodiaconou
@@ -117,6 +117,7 @@ files:
 - lib/sentiero/stores/redis/keys.rb
 - lib/sentiero/stores/redis/lua.rb
 - lib/sentiero/stores/sqlite.rb
+- lib/sentiero/stores/sqlite/payload_codec.rb
 - lib/sentiero/stores/sqlite/schema.rb
 - lib/sentiero/user_agent.rb
 - lib/sentiero/version.rb