rails-profiler 0.31.1 → 0.31.2

data/lib/profiler/middleware/streamed_profile.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require_relative "../collectors/lifecycle"
+require_relative "../current_context"
+
+module Profiler
+  module Middleware
+    # The profile of a streamed response between the moment the application returns and the
+    # moment the server closes the body. The collectors that read nothing from the request's
+    # thread (Collectors::BaseCollector#collect_from_any_thread?) stay subscribed meanwhile, so
+    # that the queries, views and logs of the stream are recorded.
+    #
+    # While the server iterates the body (enter, leave), its fiber carries the profile's token
+    # and the thread-local slots of the collectors kept, so that the logs, outbound HTTP calls
+    # and measures of the stream are recorded, and a filter by request can tell its events.
+    #
+    # Rack requires the server to close the body, but nothing here relies on it alone: a
+    # stream still open when the execution context that started it (its fiber) starts another
+    # profiled request was left behind, and its profile is finished then; one subscribed for
+    # longer than MAX_SUBSCRIBED_SECONDS has its collectors released by the next profiled
+    # request, from any thread, which the profile says. The body itself is closed only by the
+    # server. The context is the fiber, not the thread: a fiber-based server (Falcon) runs
+    # many requests at once on one thread, each in its own fiber, and Thread#[] slots are
+    # fiber-local too; a threaded server runs each request in its thread's root fiber.
+    class StreamedProfile
+      MAX_SUBSCRIBED_SECONDS = 300
+
+      @pending = {}
+      @lock = Mutex.new
+
+      class << self
+        def register(streamed)
+          @lock.synchronize { @pending[streamed.object_id] = streamed }
+        end
+
+        def unregister(streamed)
+          @lock.synchronize { @pending.delete(streamed.object_id) }
+        end
+
+        # Run when a profiled request starts.
+        def sweep(context = Fiber.current)
+          now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          left, old = @lock.synchronize do
+            [@pending.values.select { |s| s.context.equal?(context) },
+             @pending.values.select { |s| !s.context.equal?(context) && now - s.started_at > MAX_SUBSCRIBED_SECONDS }]
+          end
+          left.each(&:abandon)
+          old.each do |streamed|
+            unregister(streamed)
+            streamed.release_collectors(after_seconds: MAX_SUBSCRIBED_SECONDS)
+          end
+        end
+
+        # Releases every stream still subscribed (specs, and a process shutting down).
+        def release_all_pending
+          @lock.synchronize { @pending.values.tap { @pending.clear } }.each(&:release_collectors)
+        end
+      end
+
+      attr_reader :profile, :context, :started_at
+      attr_accessor :body
+
+      # on_finish runs once, with what the body captured, when it is closed or abandoned.
+      def initialize(profile, collectors, &on_finish)
+        @profile = profile
+        @collectors = collectors
+        @on_finish = on_finish
+        @context = Fiber.current
+        @started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        @mutex = Mutex.new
+        @released = false
+        @finished = false
+      end
+
+      # Collects the collectors kept for the stream and releases them, once, from any thread.
+      # +after_seconds+: released before the body was closed, after that long; the profile says
+      # so, as what came after is missing from it.
+      def release_collectors(after_seconds: nil)
+        @mutex.synchronize do
+          return if @released
+
+          @released = true
+          @profile.collectors_released_after_seconds = after_seconds if after_seconds
+          @collectors.each do |collector|
+            collector.collect if collector.respond_to?(:collect)
+            @profile.refresh_collector_metadata(collector)
+          rescue => e
+            warn "Collector #{collector.class} failed: #{e.message}"
+          end
+          Collectors::Lifecycle.release_all(@collectors)
+        end
+      end
+
+      def finish(captured, size, complete, error)
+        @mutex.synchronize do
+          return if @finished
+
+          @finished = true
+        end
+        self.class.unregister(self)
+        @on_finish.call(captured, size, complete, error)
+      ensure
+        # Whatever on_finish did, nothing stays subscribed.
+        release_collectors
+      end
+
+      # The server moved on without closing the body: its profile is finished with what it
+      # captured so far. The body stays the server's to close.
+      def abandon
+        body ? body.abandon : finish("", 0, false, nil)
+      end
+
+      # Called by the body around its iteration, on the fiber that iterates it. Lends nothing
+      # once the collectors are released.
+      def enter
+        token = CurrentContext.token
+        CurrentContext.token = @profile.token
+        lent = @mutex.synchronize do
+          next [] if @released
+
+          @collectors.filter_map do |collector|
+            [collector, collector.lend_thread_slots] if collector.respond_to?(:lend_thread_slots)
+          end
+        end
+        [token, lent]
+      end
+
+      def leave(state)
+        token, lent = state
+        lent.reverse_each { |collector, previous| collector.return_thread_slots(previous) }
+        CurrentContext.token = token
+      end
+    end
+  end
+end

data/lib/profiler/middleware/toolbar_injector.rb

@@ -129,17 +129,10 @@ module Profiler
         end
       end
 
+      # The middleware hands over the page it has already read: an Array of parts, or a String.
+      # Never a body to iterate, which would drain a stream and close it.
       def extract_content(body)
-        if body.respond_to?(:body)
-          body.body
-        elsif body.respond_to?(:each)
-          parts = []
-          body.each { |part| parts << part }
-          body.close if body.respond_to?(:close)
-          parts.join
-        else
-          body.to_s
-        end
+        body.is_a?(String) ? body : Array(body).join
       end
 
       def toolbar_html

data/lib/profiler/models/profile.rb

@@ -5,17 +5,21 @@ require "securerandom"
 require "json"
 require "zlib"
 require_relative "../redaction"
+require_relative "../allocation_counter"
 
 module Profiler
   module Models
     class Profile
       attr_reader :path
-      attr_accessor :token, :method, :status, :duration, :memory,
+      attr_accessor :token, :method, :status, :duration, :allocated_objects,
                     :started_at, :finished_at, :params, :headers,
                     :response_headers, :collectors_data, :collectors_metadata,
                     :parent_token, :is_ajax, :profile_type,
                     :request_body, :request_body_encoding,
                     :response_body, :response_body_encoding,
+                    :request_body_size, :request_body_truncated, :request_body_size_is_minimum,
+                    :response_body_size, :response_body_truncated, :response_body_size_is_minimum,
+                    :collectors_released_after_seconds,
                     :gem_version
 
       def initialize(request = nil)
@@ -35,13 +39,32 @@ module Profiler
         end
       end
 
-      def set_bodies(request_body:, response_body:, req_content_type:, resp_content_type:)
+      # The sizes are those of the whole bodies, when a body was cut at max_captured_body_bytes;
+      # *_size_is_minimum when the whole size is not known (no Content-Length, a stream that
+      # stopped), only that it is at least that.
+      def set_bodies(request_body:, response_body:, req_content_type:, resp_content_type:,
+                     request_body_size: nil, response_body_size: nil,
+                     request_body_size_is_minimum: false, response_body_size_is_minimum: false)
         req  = process_body(request_body, req_content_type)
         resp = process_body(response_body, resp_content_type)
         @request_body          = req[:body]
         @request_body_encoding = req[:encoding]
         @response_body         = resp[:body]
         @response_body_encoding = resp[:encoding]
+        @request_body_size, @request_body_truncated = body_size(request_body, request_body_size)
+        @response_body_size, @response_body_truncated = body_size(response_body, response_body_size)
+        @request_body_size_is_minimum = request_body_size_is_minimum ? true : false
+        @response_body_size_is_minimum = response_body_size_is_minimum ? true : false
+      end
+
+      # Deprecated: the number of allocated objects times 40, the figure earlier versions
+      # reported as bytes. Still written in to_h, for the readers that only know it.
+      def memory
+        @allocated_objects&.*(AllocationCounter::LEGACY_BYTES_PER_OBJECT)
+      end
+
+      def memory=(value)
+        @allocated_objects = value && value / AllocationCounter::LEGACY_BYTES_PER_OBJECT
       end
 
       def finish(status, response_headers = {})
@@ -67,8 +90,20 @@ module Profiler
       end
 
       def add_collector_metadata(collector)
+        @collectors_metadata << collector_metadata(collector)
+      end
+
+      # A collector collected again, after the profile's tabs were listed (a streamed response
+      # that failed part way): its tab keeps its place, with what it now has.
+      def refresh_collector_metadata(collector)
+        entry = collector_metadata(collector)
+        index = @collectors_metadata.index { |tab| tab[:key] == entry[:key] }
+        index ? @collectors_metadata[index] = entry : @collectors_metadata << entry
+      end
+
+      def collector_metadata(collector)
         config = collector.tab_config
-        @collectors_metadata << {
+        {
           key: config[:key],
           label: config[:label],
           icon: config[:icon],
@@ -79,6 +114,7 @@ module Profiler
           has_data: collector.has_data?
         }
       end
+      private :collector_metadata
 
       def to_h
         req_body,  req_enc  = decode_body(@request_body,  @request_body_encoding)
@@ -92,7 +128,8 @@ module Profiler
           method: @method,
           status: @status,
           duration: @duration,
-          memory: @memory,
+          allocated_objects: @allocated_objects,
+          memory: memory,
           started_at: @started_at&.iso8601,
           finished_at: @finished_at&.iso8601,
           params: @params,
@@ -102,6 +139,13 @@ module Profiler
           request_body_encoding: req_enc,
           response_body: resp_body,
           response_body_encoding: resp_enc,
+          request_body_size: @request_body_size,
+          request_body_truncated: @request_body_truncated,
+          request_body_size_is_minimum: @request_body_size_is_minimum,
+          response_body_size: @response_body_size,
+          response_body_truncated: @response_body_truncated,
+          response_body_size_is_minimum: @response_body_size_is_minimum,
+          collectors_released_after_seconds: @collectors_released_after_seconds,
           collectors_data: @collectors_data,
           tabs: @collectors_metadata,
           parent_token: @parent_token,
@@ -125,7 +169,12 @@ module Profiler
         profile.method = data[:method]
         profile.status = data[:status]
         profile.duration = data[:duration]
-        profile.memory = data[:memory]
+        # A profile saved before allocated_objects only carries memory, the count times 40.
+        if data.key?(:allocated_objects)
+          profile.allocated_objects = data[:allocated_objects]
+        else
+          profile.memory = data[:memory]
+        end
         profile.started_at = data[:started_at] ? Time.parse(data[:started_at]) : nil
         profile.finished_at = data[:finished_at] ? Time.parse(data[:finished_at]) : nil
         profile.params = data[:params]
@@ -135,6 +184,15 @@ module Profiler
         profile.request_body_encoding = data[:request_body_encoding] || "text"
         profile.response_body = data[:response_body]
         profile.response_body_encoding = data[:response_body_encoding] || "text"
+        profile.request_body_size = data[:request_body_size]
+        # Left nil when the store did not keep them (SqliteStore rebuilds a profile from its
+        # columns): the request collector's data still has them.
+        profile.request_body_truncated = data[:request_body_truncated]
+        profile.request_body_size_is_minimum = data[:request_body_size_is_minimum]
+        profile.response_body_size = data[:response_body_size]
+        profile.response_body_truncated = data[:response_body_truncated]
+        profile.response_body_size_is_minimum = data[:response_body_size_is_minimum]
+        profile.collectors_released_after_seconds = data[:collectors_released_after_seconds]
         profile.parent_token = data[:parent_token]
         profile.is_ajax = data[:is_ajax] || false
         profile.profile_type = data[:profile_type] || "http"
@@ -181,6 +239,12 @@ module Profiler
         end
       end
 
+      def body_size(captured, total)
+        captured_size = captured.to_s.bytesize
+        total ||= captured_size
+        [total, total > captured_size]
+      end
+
       def compress_body?(text)
         Profiler.configuration.compress_bodies &&
           text.bytesize > Profiler.configuration.compress_body_threshold

data/lib/profiler/redaction.rb

@@ -88,6 +88,16 @@ module Profiler
         text.length > max ? "#{text[0, max]}#{ellipsis}" : text
       end
 
+      # The first +max+ bytes of +text+, a body kept only in part. A credential cut in two by the
+      # limit would escape the masking by value, which runs later on what is kept: the bytes
+      # that could start one are left out as well. Text no longer than +max+ is returned whole.
+      def cut_bytes(text, max)
+        return text if text.bytesize <= max
+
+        cut = text.byteslice(0, max)
+        cut.byteslice(0, cut.bytesize - held_back_bytes(cut))
+      end
+
       # How many trailing bytes of +text+, a stream read in pieces, could be
       # the start of a credential and have to wait for the next piece: the
       # longest end of the text that is also the beginning of a credential,

data/lib/profiler/test_profiler.rb

@@ -56,7 +56,7 @@ module Profiler
 
       exception_collector = collectors.find { |c| c.is_a?(Collectors::ExceptionCollector) }
 
-      memory_before = current_memory if Profiler.configuration.track_memory
+      allocations_before = AllocationCounter.current if Profiler.configuration.track_memory
 
       test_status = "passed"
       error_message = nil
@@ -103,7 +103,7 @@ module Profiler
         Profiler::CurrentContext.token = previous_token
 
         if Profiler.configuration.track_memory
-          profile.memory = current_memory - memory_before
+          profile.allocated_objects = AllocationCounter.current - allocations_before
         end
 
         test_collector.update_status(test_status, error_message)
@@ -124,20 +124,5 @@ module Profiler
       # After collect, and also when collect or the storage failed.
       Collectors::Lifecycle.release_all(collectors)
     end
-
-    private
-
-    def current_memory
-      return 0 unless defined?(GC.stat)
-
-      stats = GC.stat
-      if stats.key?(:total_allocated_size)
-        stats[:total_allocated_size]
-      elsif stats.key?(:total_allocated_objects)
-        stats[:total_allocated_objects] * 40
-      else
-        0
-      end
-    end
   end
 end

data/lib/profiler/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Profiler
-  VERSION = "0.31.1"
+  VERSION = "0.31.2"
 end

data/lib/profiler.rb

@@ -5,6 +5,7 @@ require_relative "profiler/boot_env"
 require_relative "profiler/version"
 require_relative "profiler/configuration"
 require_relative "profiler/redaction"
+require_relative "profiler/allocation_counter"
 
 module Profiler
   class Error < StandardError; end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rails-profiler
 version: !ruby/object:Gem::Version
-  version: 0.31.1
+  version: 0.31.2
 platform: ruby
 authors:
 - Sébastien Duplessy
@@ -133,6 +133,7 @@ files:
 - config/routes.rb
 - exe/profiler-mcp
 - lib/profiler.rb
+- lib/profiler/allocation_counter.rb
 - lib/profiler/boot_env.rb
 - lib/profiler/cluster/master_client.rb
 - lib/profiler/cluster/security.rb
@@ -206,8 +207,10 @@ files:
 - lib/profiler/mcp/tools/reset_env_var.rb
 - lib/profiler/mcp/tools/run_tests.rb
 - lib/profiler/mcp/tools/set_env_var.rb
+- lib/profiler/middleware/capturing_body.rb
 - lib/profiler/middleware/cors_middleware.rb
 - lib/profiler/middleware/profiler_middleware.rb
+- lib/profiler/middleware/streamed_profile.rb
 - lib/profiler/middleware/toolbar_injector.rb
 - lib/profiler/models/profile.rb
 - lib/profiler/models/sql_query.rb
@@ -235,13 +238,13 @@ files:
 - lib/profiler/test_runner/run_store.rb
 - lib/profiler/test_runner/runner.rb
 - lib/profiler/version.rb
-homepage: https://git.duplessy.eu/sebastien/rails-profiler-gem
+homepage: https://github.com/Pepsit36/rails-profiler
 licenses:
 - MIT
 metadata:
-  homepage_uri: https://git.duplessy.eu/sebastien/rails-profiler-gem
-  source_code_uri: https://git.duplessy.eu/sebastien/rails-profiler-gem
-  changelog_uri: https://git.duplessy.eu/sebastien/rails-profiler-gem/-/blob/master/CHANGELOG.md
+  homepage_uri: https://github.com/Pepsit36/rails-profiler
+  source_code_uri: https://github.com/Pepsit36/rails-profiler
+  changelog_uri: https://github.com/Pepsit36/rails-profiler/blob/master/CHANGELOG.md
 post_install_message:
 rdoc_options: []
 require_paths: