runlight 0.1.6 → 0.1.8

data/lib/runlight/store/sql_store.rb

@@ -97,17 +97,32 @@ module Runlight
 
       # Keeps SQLite's planner statistics current, which it never gathers by itself. Without them it can
       # choose a plan that reads a table once for every row of another. A sample of each index is enough,
-      # so this takes milliseconds even on a large database. Postgres gathers its own.
-      def optimize(only_when_missing = false)
+      # so this takes a few seconds at most even on a large database. Postgres gathers its own.
+      #
+      # With only_when_stale, only when there are none, or the events have more than doubled since they were
+      # gathered. Statistics gathered on a new, empty database say every table is empty, and a goal over a
+      # million visits then took a minute rather than milliseconds.
+      def optimize(only_when_stale = false)
         return if @db.dialect != "sqlite"
 
         begin
-          return if only_when_missing && !@db.all("SELECT name FROM sqlite_master WHERE name = 'sqlite_stat1'").empty?
-
-          @db.run("PRAGMA analysis_limit = 1000")
-          @db.run("ANALYZE")
+          events = Sql.num(@db.all("SELECT MAX(id) AS n FROM rl_events")[0]&.fetch("n", nil))
+          if only_when_stale
+            seen = @db.all("SELECT value FROM rl_meta WHERE \"key\" = 'statistics_events'")[0]
+            then_events = seen.nil? ? nil : Js.number(seen["value"])
+            gathered = !@db.all("SELECT name FROM sqlite_master WHERE name = 'sqlite_stat1'").empty?
+            return if gathered && !then_events.nil? && then_events.finite? && !(events > 2 * then_events && events - then_events >= 1000)
+          end
+          begin
+            @db.run("PRAGMA analysis_limit = 1000")
+            @db.run("ANALYZE")
+          rescue StandardError
+            # Some hosted SQLite services refuse these, and gather statistics themselves.
+          end
+          # Noted even when refused, so a service that refuses is not asked again until the events double.
+          @db.run(Sql.upsert(@db.dialect, "rl_meta", ['"key"', "value"], ['"key"'], ["value"]), ["statistics_events", events.to_s])
         rescue StandardError
-          # Some hosted SQLite services refuse these, and gather statistics themselves.
+          # Statistics only help; a failure here never stops the app.
         end
       end
 
@@ -241,6 +256,14 @@ module Runlight
         window = [site, start, finish + Sql::EVENT_TAIL_MS, start, finish]
         transaction do |store|
           db = store.db
+          # MySQL keys a day's rows by no unique key, since a path is longer than its keys allow, so two processes
+          # building one day could each add its rows. There the day is marked first: the second waits on that
+          # mark until the first commits, then fails, and the day counts as built elsewhere. Elsewhere the rows'
+          # own key refuses a second set, and a database without transactions (D1) must not mark a day first.
+          if dialect == "mysql"
+            db.run("DELETE FROM rl_rollup_days WHERE site = ? AND day = ?", [site, day])
+            db.run("INSERT INTO rl_rollup_days (site, day, start_at, end_at) VALUES (?, ?, ?, ?)", [site, day, start, finish])
+          end
           db.run("DELETE FROM rl_rollups WHERE site = ? AND day = ?", [site, day])
           # The WITH goes after INSERT INTO, the one place every database takes it.
           params = [site, start, finish]
@@ -268,8 +291,10 @@ module Runlight
          SELECT ?, ?, 'event', e.name, COUNT(DISTINCT e.visitor), COUNT(*) #{of_day.call("event")} GROUP BY e.name",
             [site, day, *window],
           )
-          db.run("DELETE FROM rl_rollup_days WHERE site = ? AND day = ?", [site, day])
-          db.run("INSERT INTO rl_rollup_days (site, day, start_at, end_at) VALUES (?, ?, ?, ?)", [site, day, start, finish])
+          if dialect != "mysql"
+            db.run("DELETE FROM rl_rollup_days WHERE site = ? AND day = ?", [site, day])
+            db.run("INSERT INTO rl_rollup_days (site, day, start_at, end_at) VALUES (?, ?, ?, ?)", [site, day, start, finish])
+          end
         end
       end
 
@@ -308,7 +333,13 @@ module Runlight
 
       # When the site last recorded a visit, or nil if it never has.
       def last_seen(site)
-        row = @db.all("SELECT MAX(ts) AS t FROM rl_events WHERE site = ? AND kind IN ('pageview', 'event')", [site])[0]
+        # One MAX for each kind, which every database reads from the end of its (site, kind, ts) index. Asked for
+        # both kinds at once, MySQL read every pageview and event of the site: 2 s on a million visits.
+        row = @db.all(
+          "SELECT MAX(t) AS t FROM (SELECT MAX(ts) AS t FROM rl_events WHERE site = ? AND kind = 'pageview'
+       UNION ALL SELECT MAX(ts) AS t FROM rl_events WHERE site = ? AND kind = 'event') x",
+          [site, site],
+        )[0]
         row.nil? || row["t"].nil? ? nil : Sql.num(row["t"])
       end
 
@@ -337,9 +368,21 @@ module Runlight
         rows[0].nil? || rows[0]["salt"].nil? ? nil : Sql.string(rows[0]["salt"])
       end
 
-      # Deletes every salt older than `day`, so old hashes can never be recomputed.
-      def drop_salts_before(day)
-        @db.run("DELETE FROM rl_salts WHERE day < ?", [day])
+      # Deletes every salt but those of these days, so the hashes made with them can never be made again.
+      def drop_salts_except(days)
+        @db.run("DELETE FROM rl_salts WHERE day NOT IN (#{(["?"] * days.length).join(", ")})", days)
+      end
+
+      # Whether a visit to one of these sites that started in a stretch of time was still going at `since`.
+      def visit_open(sites, from, to, since)
+        # As many sites a statement as keep it under D1's 100.
+        open = Sql.in_pieces(sites, Sql::MAX_PARAMS - 3) do |piece|
+          @db.all(
+            "SELECT 1 AS open FROM rl_sessions WHERE site IN (#{(["?"] * piece.length).join(", ")}) AND started_at >= ? AND started_at < ? AND last_at >= ? LIMIT 1",
+            [*piece, from, to, since],
+          )
+        end
+        !open.empty?
       end
 
       # Ingest
@@ -356,17 +399,35 @@ module Runlight
         rows[0].nil? ? nil : { "id" => Sql.string(rows[0]["id"]), "visitor" => Sql.string(rows[0]["visitor"]) }
       end
 
+      # After opening a visit: the visitor's earliest visit still going, which is this one unless another
+      # process opened one for the same visitor at the same moment. Then both settle on the earliest, and the
+      # one that is not drops its own, before anything is counted into it.
+      def settle_open_session(site, visitors, since, opened)
+        rows = @db.all(
+          # One already counting a hit first: the other process went on with it.
+          "SELECT id, visitor FROM rl_sessions WHERE site = ? AND visitor IN (#{(["?"] * visitors.length).join(", ")}) AND last_at >= ?
+       ORDER BY CASE WHEN pageviews > 0 OR events > 0 THEN 0 ELSE 1 END, started_at, id LIMIT 1",
+          [site, *visitors, since],
+        )
+        first = rows[0] || opened
+        @db.run("DELETE FROM rl_sessions WHERE id = ? AND pageviews = 0 AND events = 0", [opened["id"]]) if first["id"] != opened["id"]
+        { "id" => Sql.string(first["id"]), "visitor" => Sql.string(first["visitor"]) }
+      end
+
       def insert_session(row)
+        c = Sql.method(:clip)
         @db.run(
           "INSERT INTO rl_sessions (id, site, visitor, started_at, last_at, engaged_ms, hostname, referrer_host, referrer_path,
         source, channel, utm_source, utm_medium, utm_campaign, utm_term, utm_content, country, region, city,
         browser, browser_version, os, os_version, device, screen, language)
        VALUES (?, ?, ?, ?, ?, 0, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
           [
-            row["id"], row["site"], row["visitor"], row["startedAt"], row["startedAt"], row["hostname"], row["referrerHost"], row["referrerPath"],
-            row["source"], row["channel"], row["utmSource"], row["utmMedium"], row["utmCampaign"], row["utmTerm"], row["utmContent"],
-            row["country"], row["region"], row["city"], row["browser"], row["browserVersion"], row["os"], row["osVersion"], row["device"],
-            row["screen"], row["language"],
+            # Each text cut to its column's width, as MySQL's columns would hold it, so every database keeps the
+            # same, and no header, however long, can make a row Postgres's indexes refuse.
+            row["id"], row["site"], row["visitor"], row["startedAt"], row["startedAt"], c.(row["hostname"], 255), c.(row["referrerHost"], 255), c.(row["referrerPath"], 500),
+            c.(row["source"], 200), c.(row["channel"], 100), c.(row["utmSource"], 200), c.(row["utmMedium"], 200), c.(row["utmCampaign"], 200), c.(row["utmTerm"], 200), c.(row["utmContent"], 200),
+            c.(row["country"], 16), c.(row["region"], 100), c.(row["city"], 100), c.(row["browser"], 100), c.(row["browserVersion"], 100), c.(row["os"], 100), c.(row["osVersion"], 100), c.(row["device"], 50),
+            c.(row["screen"], 50), c.(row["language"], 50),
           ],
         )
       end
@@ -380,7 +441,7 @@ module Runlight
           @db.run(
             "UPDATE rl_sessions SET pageviews = pageviews + 1, last_at = ?, exit_path = ?,
            entry_path = CASE WHEN entry_path = '' THEN ? ELSE entry_path END WHERE id = ?",
-            [ts, path, path, id],
+            [ts, Sql.clip(path, 1000), Sql.clip(path, 1000), id],
           )
         elsif reopen
           @db.run("UPDATE rl_sessions SET events = events + 1, last_at = ? WHERE id = ?", [ts, id])
@@ -425,8 +486,8 @@ module Runlight
           "INSERT INTO rl_events (site, ts, kind, visitor, session, pageview, path, hostname, title, name, props, engaged_ms, scroll, link)
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
           [
-            row["site"], row["ts"], row["kind"], row["visitor"], row["session"], row["pageview"], row["path"], row["hostname"], row["title"],
-            row["name"], props.nil? ? nil : Json.encode(props), row["engagedMs"], row["scroll"], row["link"],
+            row["site"], row["ts"], row["kind"], row["visitor"], row["session"], row["pageview"], Sql.clip(row["path"], 1000), Sql.clip(row["hostname"], 255), Sql.clip(row["title"], 500),
+            Sql.clip(row["name"], 255), props.nil? ? nil : Json.encode(props), row["engagedMs"], row["scroll"], row["link"],
           ],
         )
       end
@@ -735,21 +796,40 @@ module Runlight
       end
 
       def save_goal(g, before = nil)
+        write = lambda do |db|
+          db.run(
+            Sql.upsert(
+              db.dialect,
+              "rl_goals",
+              ["id", "site", "name", "kind", '"match"', "click_by", "value_mode", "value", "value_prop", "currency", "created_at"],
+              ["id"],
+              ["name", "kind", '"match"', "click_by", "value_mode", "value", "value_prop", "currency"],
+            ),
+            [g["id"], g["site"], g["name"], g["kind"], g["match"], g["clickBy"], g["valueMode"], g["value"], g["valueProp"], g["currency"], g["createdAt"]],
+          )
+        end
         # A click goal is counted by its name, which the tracker sends as the event
         # name. Renaming one renames its past clicks too, so its history stays.
-        if !before.nil? && before["kind"] == "click" && g["kind"] == "click" && before["name"] != g["name"]
-          @db.run("UPDATE rl_events SET name = ? WHERE site = ? AND kind = 'event' AND name = ?", [g["name"], g["site"], before["name"]])
+        return write.call(@db) unless !before.nil? && before["kind"] == "click" && g["kind"] == "click" && before["name"] != g["name"]
+
+        transaction do |store|
+          db = store.db
+          db.run("UPDATE rl_events SET name = ? WHERE site = ? AND kind = 'event' AND name = ?", [g["name"], g["site"], before["name"]])
+          # The daily rollups count events by name too, and goals read them, so they take the new name as well.
+          # A day that already has events under the new name counts both together now, so it is built again.
+          both = db.all(
+            "SELECT day FROM rl_rollups WHERE site = ? AND dim = 'event' AND value = ?
+         AND day IN (SELECT day FROM rl_rollups WHERE site = ? AND dim = 'event' AND value = ?)",
+            [g["site"], before["name"], g["site"], g["name"]],
+          ).map { |r| Sql.string(r["day"]) }
+          both.each do |day|
+            db.run("DELETE FROM rl_rollup_days WHERE site = ? AND day = ?", [g["site"], day])
+            db.run("DELETE FROM rl_rollups WHERE site = ? AND day = ?", [g["site"], day])
+          end
+          db.run("UPDATE rl_rollups SET value = ? WHERE site = ? AND dim = 'event' AND value = ?", [g["name"], g["site"], before["name"]])
+          write.call(db)
         end
-        @db.run(
-          Sql.upsert(
-            @db.dialect,
-            "rl_goals",
-            ["id", "site", "name", "kind", '"match"', "click_by", "value_mode", "value", "value_prop", "currency", "created_at"],
-            ["id"],
-            ["name", "kind", '"match"', "click_by", "value_mode", "value", "value_prop", "currency"],
-          ),
-          [g["id"], g["site"], g["name"], g["kind"], g["match"], g["clickBy"], g["valueMode"], g["value"], g["valueProp"], g["currency"], g["createdAt"]],
-        )
+        nil
       end
 
       def delete_goal(id)
@@ -809,7 +889,50 @@ module Runlight
       # per goal: each goal adds a conditional count, distinct count, and sum.
       # Keyed by goal id: {"conversions", "visitors", "revenue"}.
       def goal_totals_all(query, goals)
+        # A goal the daily rollups count exactly comes from the built days and the visits of the rest: a page
+        # by its whole path, or an event or click by name, worth nothing or a fixed amount. Visitor ids last a
+        # day, so each day's visitors add up, as they do for every other report. A pattern or a value sent with
+        # each event needs the events themselves.
+        rollable = ->(goal) { goal["valueMode"] != "prop" && !(goal["kind"] == "page" && goal["match"].include?("*")) }
+        plan = goals.any?(&rollable) ? rollup_plan(query, query["from"], query["to"]) : nil
+        return raw_goal_totals_all(query, goals) if plan.nil? || plan["days"].empty?
+
+        rolled = goals.select(&rollable)
+        out = raw_goal_totals_all(query, goals.reject(&rollable))
+        counted = {}
+        [
+          ["page", "pageviews", rolled.select { |g| g["kind"] == "page" }.map { |g| g["match"] }],
+          ["event", "events", rolled.reject { |g| g["kind"] == "page" }.map { |g| g["kind"] == "click" ? g["name"] : g["match"] }],
+        ].each do |dim, column, values|
+          rows = Sql.in_pieces(values.uniq, Sql::VALUES_PER_QUERY) do |piece|
+            @db.all(
+              "SELECT value, SUM(visitors) AS visitors, SUM(#{column}) AS n FROM rl_rollups
+           WHERE site = ? AND dim = '#{dim}' AND day IN (#{Sql::BUILT_DAYS}) AND value IN (#{piece.map { "?" }.join(", ")}) GROUP BY value",
+              [query["site"], query["site"], query["from"], query["to"], *piece],
+            )
+          end
+          rows.each { |row| counted["#{dim}:#{Sql.string(row["value"])}"] = { "visitors" => Sql.num(row["visitors"]), "n" => Sql.num(row["n"]) } }
+        end
+        rest = plan["rest"].map { |from, to| raw_goal_totals_all(query.merge("from" => from, "to" => to), rolled) }
+        rolled.each do |goal|
+          key = goal["kind"] == "page" ? "page:#{goal["match"]}" : "event:#{goal["kind"] == "click" ? goal["name"] : goal["match"]}"
+          sum = { "conversions" => counted.dig(key, "n") || 0, "visitors" => counted.dig(key, "visitors") || 0 }
+          rest.each do |part|
+            sum["visitors"] += part[goal["id"]]["visitors"]
+            sum["conversions"] += part[goal["id"]]["conversions"]
+          end
+          revenue = goal["valueMode"] == "fixed" ? cents(sum["conversions"] * goal["value"]) : 0
+          out[goal["id"]] = sum.merge("revenue" => revenue)
+        end
+        # In the order the goals came in, as the single pass answers.
+        goals.to_h { |goal| [goal["id"], out[goal["id"]]] }
+      end
+
+      # Every goal's totals read from the events of the range's visits.
+      def raw_goal_totals_all(query, goals)
         out = {}
+        return out if goals.empty?
+
         v = Sql.visit_rows(query["filters"], query["site"], query["from"], query["to"], @db.dialect)
         # As many goals per query as keep it under D1's parameter limit.
         chunks = [[]]
@@ -1003,9 +1126,14 @@ module Runlight
 
       # Just the visitor count from stats(), in one query, for conversion rates.
       def visitors(query)
+        # Built days add up their visitors, as the stats do.
+        rolled = rolled_stats(query)
+        return rolled["visitors"] if rolled
+
         scope = Sql.visit_scope(query["filters"], query["site"], query["from"], query["to"], @db.dialect)
+        table = @db.dialect == "sqlite" && query["filters"].empty? ? "rl_sessions s INDEXED BY rl_sessions_visit_visitors" : "rl_sessions s"
         row = @db.all(
-          "SELECT COUNT(DISTINCT s.visitor) AS visitors FROM rl_sessions s WHERE s.site = ? AND s.started_at >= ? AND s.started_at < ? AND #{Sql::IS_VISIT}#{scope["sql"]}",
+          "SELECT COUNT(DISTINCT s.visitor) AS visitors FROM #{table} WHERE s.site = ? AND s.started_at >= ? AND s.started_at < ? AND #{Sql::IS_VISIT}#{scope["sql"]}",
           [query["site"], query["from"], query["to"], *scope["params"]],
         )[0] || {}
         Sql.num(row["visitors"])
@@ -1114,6 +1242,29 @@ module Runlight
         end
       end
 
+      # Seek one matching event per hostname, skipping its remaining events before the next seek.
+      def hostname_suggestions(query, limit)
+        dialect = @db.dialect
+        v = Sql.visit_rows(query["filters"], query["site"], query["from"], query["to"], dialect)
+        within = Sql.row_scope(query["filters"], %w[page hostname], dialect)
+        table = dialect == "sqlite" ? "rl_events e INDEXED BY rl_events_hostname_suggest" : "rl_events e"
+        rows = []
+        after = nil
+        while rows.length < limit
+          found = @db.all(
+            "SELECT e.hostname AS value FROM #{table} CROSS JOIN rl_sessions s
+         WHERE #{v["sql"]} AND e.kind = 'pageview'#{within["sql"]}#{after.nil? ? "" : " AND e.hostname#{text_order} > ?"}
+         ORDER BY e.hostname#{text_order} LIMIT 1",
+            [*v["params"], *within["params"], *(after.nil? ? [] : [after])],
+          )
+          break if found.empty?
+
+          after = Sql.string(found[0]["value"])
+          rows << { "value" => after }
+        end
+        rows
+      end
+
       # BreakdownRow list.
       def breakdown(query, dimension, limit, offset)
         page = [limit, offset]
@@ -1289,8 +1440,9 @@ module Runlight
           [site, since],
         )[0] || {}
         pages = @db.all(
-          "SELECT path AS value, COUNT(DISTINCT visitor) AS visitors FROM rl_events
-       WHERE site = ? AND ts >= ? AND kind = 'pageview' GROUP BY path ORDER BY visitors DESC, path#{text_order} LIMIT 10",
+          # Through the kind and time index: SQLite otherwise reads the site's whole history by path to group it.
+          "SELECT e.path AS value, COUNT(DISTINCT e.visitor) AS visitors FROM #{Sql.event_kind_table(@db.dialect)}
+       WHERE e.site = ? AND e.ts >= ? AND e.kind = 'pageview' GROUP BY e.path ORDER BY visitors DESC, e.path#{text_order} LIMIT 10",
           [site, since],
         )
         sources = @db.all(

data/lib/runlight/version.rb

@@ -4,7 +4,7 @@ require "json"
 
 module Runlight
   # The gem's own version, which the release keeps equal to the npm package's.
-  GEM_VERSION = "0.1.6"
+  GEM_VERSION = "0.1.8"
 
   # The SDK's version and the HTTP API's, read from assets/build.json, which
   # scripts/ruby-assets.mts writes from the TypeScript SDK, so the two always

data/lib/runlight/zip.rb

@@ -42,7 +42,9 @@ module Runlight
       values.map do |v|
         s = v.nil? || v.equal?(UNDEFINED) ? "" : Js.string(v)
         s = "'#{s}" if s.match?(/\A[=+\-@\t\r]/) && !s.match?(/\A-?\d+(\.\d+)?\z/)
-        s.match?(/[",\n\r]/) ? "\"#{s.gsub('"', '""')}\"" : s
+        # Quoted with a semicolon or tab in it too: Excel in a locale that separates lists with ";" splits an
+        # unquoted cell there, and what follows could start with = and run as a formula.
+        s.match?(/[",;\t\n\r]/) ? "\"#{s.gsub('"', '""')}\"" : s
       end.join(",")
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: runlight
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.8
 platform: ruby
 authors:
 - Jon C. Phillips
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2026-10-14 00:00:00.000000000 Z
+date: 2026-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord