rails-profiler 0.33.2 → 0.33.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8e1caccb1662a8f09108682f5933e943d41a0b36ed7d60930c60eb8c385b5159
-  data.tar.gz: c10d767853cc07caa36484f9693a4c4adeef415e0874d29ca817d8d7d1b17730
+  metadata.gz: 533fac499cbab38112f1be4ab7d066761f0f0cea1b22d76e4d355059ef03d9dd
+  data.tar.gz: b8d37a14501188a671be2ee4876b8ce9678641cca8acbd836e456ea62249ef48
 SHA512:
-  metadata.gz: 8aacfc0469f52d9f6a11a58356eebb853ff716f2729ca47833b9b0ca99197f767a0dad5904f8c9e7d8a8501869dd712a0c5e6e246fa8ea0e55976b546589d1ac
-  data.tar.gz: 3ce33e8db95c0ae4028a68e11d941bb72072b5a6edcf539cfc0fb918c2248d675c4df92e28788da0d59d942546458384b22ec5ac974bb58404be1d0049130627
+  metadata.gz: 7daad08c5c0b6a119933341e7b1a1ba28d746c2801eef525a83dd54388c85a30bfa68a7240d2e99705992e622929df012b98676f0ec396a028fff759690910da
+  data.tar.gz: d205ba6d72f09ca00d1eb714db57f99052e64297caddb33999c8c6ae87ae380874cf015af986d0065d7b76c949388a813f3f397adae3fe23b7f5a1e6bdbcbc48

data/CHANGELOG.md

@@ -19,6 +19,35 @@ every commit of every tag interval is accounted for one way or the other.
 
 ## [Unreleased]
 
+## [0.33.3] - 2026-10-14
+
+<!-- stamped -->
+
+### Fixed
+
+- **Rake tasks:** `rake profiler:cleanup` with an empty or non-numeric `OLDER_THAN` (`""`,
+  `abc`) no longer removes every profile, and `OLDER_THAN=1h` is no longer read as one second.
+  `OLDER_THAN` and `LIMIT` take a whole number only; any other value stops the task with an error
+  and removes nothing. `OLDER_THAN=0` still removes every profile, when written so.
+- **Storage:** The SQLite store compares the start times of the profiles in UTC. Outside UTC,
+  `rake profiler:cleanup` and `cleanup` removed recent profiles (America/New_York) or kept old
+  ones (Asia/Tokyo), and the lists were out of order across a clock change. The profiles already
+  in the database are converted once, when the store opens it; the API shows the same times.
+- **Storage:** The Redis store takes expired profiles out of its lists at each save, and its list
+  keys now expire a TTL after the last save. Without `max_profiles`, or between two evictions,
+  the tokens of expired profiles stayed listed until a manual cleanup, and a page of the list
+  came back with fewer profiles than asked.
+- **Storage:** With the SQLite store, a profile saved again without a response body no longer
+  shows the response body of its previous save. The response bodies left on disk by profiles
+  removed outside the store are removed by `cleanup` and when the oldest profiles are evicted.
+
+### Limitations
+
+- **Storage:** Two SQLite stores that share a `blob_path` remove each other's response bodies:
+  give each `database` its own `blob_path` (see `docs/storage.md`).
+- **Storage:** On the first save after an upgrade, the Redis store can drop from its lists a job
+  that started more than a TTL ago and was saved recently. It happens once.
+
 ## [0.33.2] - 2026-10-14
 
 <!-- stamped -->

data/lib/profiler/storage/blob_store.rb

@@ -2,6 +2,7 @@
 
 require "fileutils"
 require "json"
+require "set"
 require_relative "private_files"
 require_relative "token"
 
@@ -12,6 +13,8 @@ module Profiler
     # find nothing and writes are refused.
     class BlobStore
       NAME_FORMAT = /\A[a-z0-9_]+\z/
+      # A save writes its blobs before its row: prune leaves the directories this recent alone.
+      PRUNE_GRACE = 60
 
       def initialize(path)
         @path = path.to_s
@@ -46,6 +49,36 @@ module Profiler
         FileUtils.rm_rf(dir) if File.directory?(dir)
       end
 
+      # Removes one blob of token, and its directory once it holds no other.
+      def delete_blob(token, collector_name)
+        return unless valid?(token, collector_name)
+
+        dir = token_dir(token)
+        FileUtils.rm_f(File.join(dir, "#{collector_name}.json"))
+        Dir.rmdir(dir) if File.directory?(dir) && Dir.empty?(dir)
+      rescue SystemCallError => e
+        Profiler.log_error("BlobStore: could not delete #{token}/#{collector_name}", e)
+      end
+
+      # Removes the directories of the tokens not in except_tokens: those whose row was deleted
+      # outside the store, or that a failed save left behind. Only the directories named as a
+      # token and unchanged for grace seconds are considered.
+      def prune(except_tokens, grace: PRUNE_GRACE)
+        keep = except_tokens.to_set
+        cutoff = Time.now - grace
+        Dir.each_child(@path) do |name|
+          next if !Token.valid?(name) || keep.include?(name)
+
+          dir = token_dir(name)
+          stat = File.lstat(dir)
+          FileUtils.rm_rf(dir) if stat.directory? && stat.mtime < cutoff
+        rescue SystemCallError => e
+          Profiler.log_error("BlobStore: could not prune #{name}", e)
+        end
+      rescue SystemCallError => e
+        Profiler.log_error("BlobStore: could not prune #{@path}", e)
+      end
+
       def exists?(token, collector_name)
         return false unless valid?(token, collector_name)
 

data/lib/profiler/storage/redis_store.rb

@@ -28,6 +28,9 @@ module Profiler
       # of an earlier version) is removed by one process, under a lock, by batches of INDEX_BATCH.
       EVICT_BATCH = 100
       EVICT_LOCK_SECONDS = 60
+      # A page that held expired profiles is read again, once they are out of the indexes, at
+      # most this many times.
+      LIST_REPAIRS = 3
       # Deletes a lock only while it still holds the value its owner set: one that expired and was
       # taken by another process is left to it.
       RELEASE_LOCK = <<~LUA
@@ -50,7 +53,8 @@ module Profiler
         data = profile.to_h
         unindex(token, read_summary(token))
         @redis.setex(profile_key(token), @ttl, data.to_json)
-        index(token, profile.started_at.to_f, Summary.build(data))
+        @redis.pipelined { |pipe| index(token, profile.started_at.to_f, Summary.build(data), client: pipe) }
+        prune_expired
         evict_first_saved(token)
         token
       end
@@ -68,19 +72,18 @@ module Profiler
         nil
       end
 
-      # Newest first. summary: true reads the summaries only.
+      # Newest first. summary: true reads the summaries only. The profiles found expired are taken
+      # out of the indexes and the page read again, for it to hold limit profiles when there are.
       def list(limit: 50, offset: 0, type: nil, summary: false)
         ensure_index
-        tokens = @redis.zrevrange(type ? type_list_key(type) : list_key, offset, offset + limit - 1)
-        return tokens.filter_map { |token| load(token) } unless summary
+        key = type ? type_list_key(type) : list_key
+        LIST_REPAIRS.times do
+          page, expired = list_page(key, offset, limit, summary)
+          return page if expired.empty?
 
-        summaries = tokens.empty? ? [] : @redis.mget(*tokens.map { |token| summary_key(token) })
-        tokens.zip(summaries).filter_map do |token, json|
-          next Summary.to_profile(JSON.parse(json)) if json
-
-          profile = load(token)
-          profile && Summary.to_profile(Summary.build(profile))
+          remove_tokens(expired)
         end
+        list_page(key, offset, limit, summary).first
       end
 
       def cleanup(older_than: 24 * 60 * 60)
@@ -118,6 +121,46 @@ module Profiler
         Redis.new(url: url)
       end
 
+      # The page at offset, and the tokens of it whose profile has expired.
+      def list_page(key, offset, limit, summary)
+        tokens = @redis.zrevrange(key, offset, offset + limit - 1)
+        return [[], []] if tokens.empty?
+
+        expired = []
+        page = if summary
+          summaries = @redis.mget(*tokens.map { |token| summary_key(token) })
+          tokens.zip(summaries).filter_map do |token, json|
+            next Summary.to_profile(JSON.parse(json)) if json
+
+            profile = load(token)
+            expired << token if profile.nil? && !@redis.exists?(profile_key(token))
+            profile && Summary.to_profile(Summary.build(profile))
+          end
+        else
+          profiles = @redis.mget(*tokens.map { |token| profile_key(token) })
+          tokens.zip(profiles).filter_map do |token, json|
+            if json.nil?
+              expired << token
+              next
+            end
+
+            Models::Profile.from_json(json)
+          rescue => e
+            Profiler.log_error("RedisStore: could not load profile #{token}", e)
+            nil
+          end
+        end
+        [page, expired]
+      end
+
+      # A profile expires a TTL after its last save, with its keys, and Redis says nothing to the
+      # indexes: the tokens saved before that are taken out of them at each save. The indexes
+      # themselves expire a TTL after the last save, when every profile they list has.
+      def prune_expired
+        expired = @redis.zrangebyscore(saved_key, "-inf", Time.now.to_f - @ttl)
+        expired.each_slice(INDEX_BATCH) { |tokens| remove_tokens(tokens) }
+      end
+
       # score: the start time, the order of the lists. saved_at: the order of the saves, which the
       # eviction follows.
       def index(token, score, summary, saved_at: Time.now.to_f, client: @redis)
@@ -128,6 +171,7 @@ module Profiler
         client.zadd(type_list_key(type), score, token)
         client.zadd(saved_key, saved_at, token)
         client.sadd(types_key, type)
+        [list_key, type_list_key(type), saved_key, types_key].each { |key| client.expire(key, @ttl) }
         return unless Token.valid?(parent)
 
         client.sadd(children_key(parent), token)

data/lib/profiler/storage/sqlite_store.rb

@@ -2,6 +2,7 @@
 
 require "json"
 require "fileutils"
+require "time"
 require_relative "base_store"
 require_relative "blob_store"
 require_relative "private_files"
@@ -14,6 +15,7 @@ module Profiler
     class SqliteStore < BaseStore
       # Past max_profiles, the oldest profiles are evicted down to this share of it.
       LOW_WATER = 0.8
+      HTTP_BODIES = "http_response_bodies"
 
       def initialize(options = {})
         require "sqlite3"
@@ -40,13 +42,14 @@ module Profiler
         data = profile.to_h
 
         collectors_meta = {}
+        bodies_saved = false
         (data[:collectors_data] || {}).each do |collector_name, collector_data|
           next unless collector_data.is_a?(Hash)
 
           if collector_name.to_s == "http"
             requests = collector_data["requests"]
             if requests
-              stripped = save_http_response_bodies(token, requests)
+              stripped, bodies_saved = save_http_response_bodies(token, requests)
               collectors_meta[collector_name] = collector_data.merge("requests" => stripped)
             else
               collectors_meta[collector_name] = collector_data
@@ -55,6 +58,9 @@ module Profiler
             collectors_meta[collector_name] = collector_data
           end
         end
+        # A profile saved again without response bodies keeps none of its previous save: they
+        # would be merged into its requests on load.
+        @blob_store.delete_blob(token, HTTP_BODIES) unless bodies_saved
 
         @db.execute(
           <<~SQL,
@@ -76,8 +82,8 @@ module Profiler
           status:           data[:status],
           duration:         data[:duration],
           memory:           data[:memory],
-          started_at:       data[:started_at],
-          finished_at:      data[:finished_at],
+          started_at:       utc_iso8601(data[:started_at]),
+          finished_at:      utc_iso8601(data[:finished_at]),
           parent_token:     data[:parent_token],
           is_ajax:          data[:is_ajax] ? 1 : 0,
           tabs:             JSON.generate(data[:tabs] || []),
@@ -157,6 +163,7 @@ module Profiler
         ).map { |r| r["token"] }
         @db.execute("DELETE FROM profiler_profiles WHERE started_at < :cutoff", cutoff: cutoff)
         tokens.each { |token| @blob_store.delete(token) }
+        prune_blobs
       end
 
       private
@@ -179,6 +186,12 @@ module Profiler
           @db.execute("DELETE FROM profiler_profiles WHERE token IN (#{(["?"] * slice.size).join(",")})", slice)
         end
         tokens.each { |token| @blob_store.delete(token) }
+        prune_blobs
+      end
+
+      # The blobs of the rows deleted outside the store (a database removed or recreated).
+      def prune_blobs
+        @blob_store.prune(@db.execute("SELECT token FROM profiler_profiles").map { |r| r["token"] })
       end
 
       # A row written before the summary column falls back to the whole row, summarized.
@@ -195,24 +208,26 @@ module Profiler
         nil
       end
 
+      # The requests without their response bodies, and whether a blob of bodies was written.
       def save_http_response_bodies(token, requests)
         bodies = requests.map do |req|
           { "response_body" => req["response_body"], "response_body_encoding" => req["response_body_encoding"] }
         end
 
-        return requests unless bodies.any? { |b| !b["response_body"].nil? }
+        return [requests, false] unless bodies.any? { |b| !b["response_body"].nil? }
 
-        @blob_store.write(token, "http_response_bodies", bodies)
+        @blob_store.write(token, HTTP_BODIES, bodies)
 
-        requests.map do |req|
+        stripped = requests.map do |req|
           req.reject { |k, _| %w[response_body response_body_encoding].include?(k) }
         end
+        [stripped, true]
       end
 
       def load_http_response_bodies(token, requests)
         return requests if requests.nil? || requests.empty?
 
-        bodies = @blob_store.read(token, "http_response_bodies")
+        bodies = @blob_store.read(token, HTTP_BODIES)
         return requests if bodies.nil? || bodies.empty?
 
         requests.each_with_index.map do |req, i|
@@ -266,6 +281,26 @@ module Profiler
         rescue SQLite3::Exception
           # column already exists
         end
+
+        migrate_times_to_utc
+      end
+
+      # Version 1 stores the times in UTC. The rows written before it carry the local offset of
+      # the process that saved them, and the default database outlives a restart: they are
+      # rewritten once, by SQLite, which reads the offset. A value it cannot read is left as is.
+      def migrate_times_to_utc
+        return if @db.get_first_value("PRAGMA user_version").to_i >= 1
+
+        @db.transaction do
+          %w[started_at finished_at].each do |column|
+            @db.execute(<<~SQL)
+              UPDATE profiler_profiles
+                 SET #{column} = strftime('%Y-%m-%dT%H:%M:%SZ', #{column})
+               WHERE #{column} NOT LIKE '%Z' AND strftime('%Y-%m-%dT%H:%M:%SZ', #{column}) IS NOT NULL
+            SQL
+          end
+          @db.execute("PRAGMA user_version = 1")
+        end
       end
 
       def row_to_profile(row, load_blobs: true)
@@ -293,8 +328,8 @@ module Profiler
           status:                 row["status"],
           duration:               row["duration"],
           memory:                 row["memory"],
-          started_at:             row["started_at"],
-          finished_at:            row["finished_at"],
+          started_at:             local_iso8601(row["started_at"]),
+          finished_at:            local_iso8601(row["finished_at"]),
           parent_token:           row["parent_token"],
           is_ajax:                row["is_ajax"] == 1,
           tabs:                   parse_json(row["tabs"], []),
@@ -312,6 +347,25 @@ module Profiler
         nil
       end
 
+      # The times are stored in UTC, all in the same form, for the cleanup and the lists to compare
+      # them as text; a profile gives them with the offset of the process that recorded it.
+      def utc_iso8601(value)
+        return value if value.nil?
+
+        Time.iso8601(value.to_s).utc.iso8601
+      rescue ArgumentError
+        value
+      end
+
+      # Read back in local time, as the profile gave them.
+      def local_iso8601(value)
+        return value if value.nil?
+
+        Time.iso8601(value).localtime.iso8601
+      rescue ArgumentError
+        value
+      end
+
       def parse_json(value, default)
         return default if value.nil? || value.empty?
 

data/lib/profiler/tasks/profiler.rake

@@ -1,17 +1,33 @@
 # frozen_string_literal: true
 
+module Profiler
+  module Tasks
+    # The whole number an environment variable gives, or default when it is not set. An empty or
+    # non-numeric value ("", "abc", "1h") stops the task: read as 0, OLDER_THAN would delete every
+    # profile.
+    def self.integer_env(name, default, minimum:, unit:)
+      value = ENV[name]
+      return default if value.nil?
+      return Integer(value, 10) if value.match?(/\A\d+\z/) && Integer(value, 10) >= minimum
+
+      warn "#{name} must be a whole number of #{unit} (#{minimum} or more), got #{value.inspect}."
+      exit 1
+    end
+  end
+end
+
 namespace :profiler do
-  desc "Clean up old profiles"
+  desc "Clean up old profiles (OLDER_THAN: seconds, 86400 by default)"
   task cleanup: :environment do
-    older_than = ENV["OLDER_THAN"]&.to_i || 24 * 60 * 60
+    older_than = Profiler::Tasks.integer_env("OLDER_THAN", 24 * 60 * 60, minimum: 0, unit: "seconds")
     puts "Cleaning up profiles older than #{older_than} seconds..."
     Profiler.storage.cleanup(older_than: older_than)
     puts "Cleanup complete!"
   end
 
-  desc "List recent profiles"
+  desc "List recent profiles (LIMIT: 20 by default)"
   task list: :environment do
-    limit = ENV["LIMIT"]&.to_i || 20
+    limit = Profiler::Tasks.integer_env("LIMIT", 20, minimum: 1, unit: "profiles")
     profiles = Profiler.storage.list(limit: limit)
 
     if profiles.empty?

data/lib/profiler/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Profiler
-  VERSION = "0.33.2"
+  VERSION = "0.33.3"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rails-profiler
 version: !ruby/object:Gem::Version
-  version: 0.33.2
+  version: 0.33.3
 platform: ruby
 authors:
 - Sébastien Duplessy