rails-minify 0.2.0 → 0.4.0

data/lib/rails_minify/sql_profile.rb

@@ -0,0 +1,236 @@
+# frozen_string_literal: true
+
+module RailsMinify
+  # Where a page's Active Record time goes, from production traffic
+  # (config.rails_minify.sql_profile = 0.01 samples 1% of requests).
+  #
+  # For each sampled request it logs one line per distinct query and call
+  # site (how many times, total time, rows, the first line in the app that
+  # ran it, the SQL with its values taken out) and one line with the
+  # request's totals: SQL time against the time spent turning rows into
+  # records. bin/rails minify:sql:report LOG=... ranks the queries by their
+  # total time; with EXPLAIN=1 it also asks the database (SQLite) for each
+  # one's plan and flags full table scans and sorts without an index.
+  module SqlProfile
+    TAG = "[rails-minify] sql"
+    MAX_SQL = 2000
+    OWN_DIR = __dir__
+
+    class << self
+      attr_reader :rate, :logger
+
+      def install!(rate:, logger: nil)
+        @rate = rate == true ? 1.0 : rate
+        @logger = logger
+        return if @installed
+
+        @installed = true
+        ActiveSupport.on_load(:action_controller_base) { prepend ControllerHook }
+        ActiveSupport::Notifications.monotonic_subscribe("sql.active_record") do |_, start, finish, _, payload|
+          current&.sql((finish - start) * 1000.0, payload)
+        end
+        ActiveSupport::Notifications.monotonic_subscribe("instantiation.active_record") do |_, start, finish, _, payload|
+          current&.instantiation((finish - start) * 1000.0, payload)
+        end
+      end
+
+      def current = ActiveSupport::IsolatedExecutionState[:rails_minify_sql_profile]
+
+      def sample? = rate && (rate >= 1.0 || rand < rate)
+
+      def track(action)
+        request = Request.new(action)
+        ActiveSupport::IsolatedExecutionState[:rails_minify_sql_profile] = request
+        yield
+      ensure
+        ActiveSupport::IsolatedExecutionState[:rails_minify_sql_profile] = nil
+        lines = request&.log_lines
+        target = logger || ActiveRecord::Base.logger
+        lines&.each { target&.info(it) }
+      end
+
+      # The SQL with its literal values replaced by ?, so the same query
+      # with other values is the same line.
+      def fingerprint(sql)
+        sql = sql.gsub(%r{/\*.*?\*/}m, " ") # query log comments
+        sql = sql.gsub(/'(?:[^']|'')*'/, "?")
+        sql = sql.gsub(/(?<![\w"`.$])-?\d+(?:\.\d+)?\b/, "?")
+        sql = sql.gsub(/\b(IN\s*)\(\s*\?(?:\s*,\s*\?)*\s*\)/i, '\1(?)')
+        sql = sql.gsub(/\s+/, " ").strip
+        sql.length > MAX_SQL ? "#{sql[0, MAX_SQL]}..." : sql
+      end
+
+      # The first line in the app (as Rails' backtrace cleaner sees it, so
+      # not in a gem) that led to the query, without the method name.
+      def call_site
+        cleaner = Rails.backtrace_cleaner
+        Thread.each_caller_location do |location|
+          next if location.path.start_with?(OWN_DIR)
+
+          frame = cleaner.clean_frame(location)
+          return frame.sub(/:in .*\z/, "").delete_prefix("./") if frame
+        end
+        "-"
+      end
+    end
+
+    Query = Struct.new(:name, :site, :sql, :count, :ms, :rows)
+
+    class Request
+      IGNORED = %w[SCHEMA TRANSACTION].freeze
+
+      def initialize(action)
+        @action = action
+        @queries = {}
+        @sql_ms = 0.0
+        @count = 0
+        @instantiation_ms = 0.0
+        @records = 0
+      end
+
+      def sql(ms, payload)
+        return if payload[:cached] || IGNORED.include?(payload[:name])
+
+        @sql_ms += ms
+        @count += 1
+        sql = SqlProfile.fingerprint(payload[:sql].to_s)
+        site = SqlProfile.call_site
+        query = (@queries[[sql, site]] ||= Query.new(payload[:name] || "SQL", site, sql, 0, 0.0, 0))
+        query.count += 1
+        query.ms += ms
+        query.rows += payload[:row_count].to_i
+      end
+
+      def instantiation(ms, payload)
+        @instantiation_ms += ms
+        @records += payload[:record_count].to_i
+      end
+
+      def log_lines
+        return [] if @count.zero?
+
+        totals = format("%s %s total queries=%d sql=%.1f instantiation=%.1f records=%d",
+          TAG, @action, @count, @sql_ms, @instantiation_ms, @records)
+        queries = @queries.values.map do |q|
+          format("%s %s query %dx %.2fms rows=%d %s | %s | %s", TAG, @action, q.count, q.ms, q.rows, q.site, q.name, q.sql)
+        end
+        [totals, *queries]
+      end
+    end
+
+    module ControllerHook
+      def process_action(...)
+        return super unless SqlProfile.sample?
+
+        SqlProfile.track("#{self.class.name}##{action_name}") { super }
+      end
+    end
+
+    # Adds up the log lines from sampled requests.
+    class Report
+      Total = Data.define(:action, :count, :sql_ms, :instantiation_ms, :records)
+      Line = Data.define(:action, :count, :ms, :rows, :site, :name, :sql)
+      Plan = Data.define(:lines, :flags, :error)
+
+      TOTAL = /#{Regexp.escape(TAG)} (\S+) total queries=(\d+) sql=([\d.]+) instantiation=([\d.]+) records=(\d+)/
+      QUERY = /#{Regexp.escape(TAG)} (\S+) query (\d+)x ([\d.]+)ms rows=(\d+) (\S+) \| (.*?) \| (.*)$/
+
+      attr_reader :totals, :lines
+
+      def initialize(lines)
+        @totals = []
+        @lines = []
+        lines.each do |line|
+          line = line.scrub.chomp
+          if (m = TOTAL.match(line))
+            @totals << Total.new(m[1], m[2].to_i, m[3].to_f, m[4].to_f, m[5].to_i)
+          elsif (m = QUERY.match(line))
+            @lines << Line.new(m[1], m[2].to_i, m[3].to_f, m[4].to_i, m[5], m[6], m[7])
+          end
+        end
+      end
+
+      # explain: a callable taking a Line and returning a Plan, or nil.
+      def report(io = $stdout, queries: 20, actions: 10, explain: nil)
+        if @totals.empty?
+          io.puts "no #{TAG} lines: set config.rails_minify.sql_profile (e.g. 0.01) and collect the production log"
+          return
+        end
+
+        sql_all = @totals.sum(&:sql_ms)
+        io.puts format("%d sampled requests: SQL %.1f ms, instantiation %.1f ms, %.0f queries, %.0f records per request",
+          @totals.size, sql_all / @totals.size, mean(@totals.map(&:instantiation_ms)), mean(@totals.map(&:count)), mean(@totals.map(&:records)))
+
+        io.puts
+        io.puts "Actions by SQL time (share of all sampled SQL time)"
+        @totals.group_by(&:action).max_by(actions) { |_, ts| ts.sum(&:sql_ms) }.each do |action, ts|
+          io.puts format("  %5.1f%%  %-45s n %4d  SQL %6.1f  instantiation %5.1f ms  queries %5.1f  records %6.0f",
+            share(ts.sum(&:sql_ms), sql_all), action, ts.size, mean(ts.map(&:sql_ms)), mean(ts.map(&:instantiation_ms)),
+            mean(ts.map(&:count)), mean(ts.map(&:records)))
+        end
+
+        requests = @totals.group_by(&:action).transform_values(&:size)
+        io.puts
+        io.puts "Queries by total time"
+        @lines.group_by(&:sql).max_by(queries) { |_, ls| ls.sum(&:ms) }.each.with_index(1) do |(sql, ls), i|
+          calls = ls.sum(&:count)
+          per_request = calls.to_f / [ls.map(&:action).uniq.sum { requests.fetch(it, 0) }, 1].max
+          io.puts
+          io.puts format("%2d. %5.1f%% of SQL time  %s  %.2f ms per call, %.1f calls per request, %.1f rows per call",
+            i, share(ls.sum(&:ms), sql_all), ls.first.name, ls.sum(&:ms) / calls, per_request, ls.sum(&:rows).to_f / calls)
+          io.puts "    #{sql}"
+          ls.group_by(&:site).max_by(3) { |_, s| s.sum(&:ms) }.each do |site, s|
+            io.puts format("    at %s (%s)", site, s.map(&:action).tally.max_by(2) { _2 }.map { "#{_1} #{_2}x" }.join(", "))
+          end
+          next unless explain
+
+          plan = explain.call(ls.first)
+          if plan.error
+            io.puts "    plan: #{plan.error}"
+          else
+            plan.lines.each { io.puts "    plan: #{it}" }
+            plan.flags.each { io.puts "    ! #{it}" }
+          end
+        end
+        io.puts
+        io.puts "Calls per request: per sampled request of the actions it runs in. Many calls of one query from one line is " \
+                "an N+1 (preload or includes); a full table scan wants an index. Instantiation is turning rows into " \
+                "records: high there, with many records, means a page loads more rows than it shows."
+      end
+
+      # EXPLAIN QUERY PLAN for SELECTs, on the connection of the model the
+      # query's name points to ("Spot Load"). SQLite only.
+      def self.sqlite_explainer
+        lambda do |line|
+          next Plan.new([], [], "not a SELECT") unless line.sql.match?(/\A\s*(WITH|SELECT)\b/i)
+
+          model = line.name.split.first.to_s.safe_constantize
+          model = ActiveRecord::Base unless model.is_a?(Class) && model < ActiveRecord::Base
+          model.with_connection do |c|
+            next Plan.new([], [], "#{c.adapter_name}: only SQLite plans are read") unless c.adapter_name.match?(/sqlite/i)
+
+            details = c.select_rows("EXPLAIN QUERY PLAN #{line.sql}").map(&:last)
+            Plan.new(details, flags(details), nil)
+          end
+        rescue StandardError => e
+          Plan.new([], [], "#{e.class}: #{e.message.lines.first&.strip}")
+        end
+      end
+
+      def self.flags(details)
+        details.filter_map do |d|
+          if (table = d[/\ASCAN (\S+)\z/, 1])
+            "full table scan of #{table}"
+          elsif d.start_with?("USE TEMP B-TREE")
+            "sorted or grouped without an index (#{d.delete_prefix("USE TEMP B-TREE FOR ").downcase})"
+          end
+        end
+      end
+
+      private
+
+      def mean(xs) = xs.empty? ? 0.0 : xs.sum.to_f / xs.size
+      def share(part, all) = all.zero? ? 0.0 : part * 100.0 / all
+    end
+  end
+end

data/lib/rails_minify/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RailsMinify
-  VERSION = "0.2.0"
+  VERSION = "0.4.0"
 end

data/lib/rails_minify.rb

@@ -9,15 +9,11 @@ rescue LoadError => e
   raise unless e.path == "ap_minimizer"
 end
 
-# One switch:
+# Strict mode, in every environment, so a call to a removed method fails in CI:
 #
-#   # config/environments/production.rb
-#   config.rails_minify.enabled = true
+#   config.ar_minimizer.slim = true   # only readers and writers per column
 #
-# turns on, unless set otherwise on their own:
-#   config.ar_minimizer.slim   strict mode: only readers and writers per column
-#   config.ar_minimizer.enums  enum predicates/bang methods as plain methods
-#   config.rails_minify.zjit   ZJIT after boot
+# ZJIT after boot is Rails' own config.zjit (Rails main).
 #
 # Experimental, each off unless set: the other minimizer options
 # (ar_minimizer.views, .load_time, .small_records, as_minimizer.unroll_callbacks,
@@ -27,6 +23,16 @@ end
 #   :omniauth                OmniAuth strategies skip their per-request dup off their paths
 #   :request_host            ActionDispatch::Request#host without a substitution per call
 module RailsMinify
+  autoload :AB, "rails_minify/ab"
+  autoload :AsyncQueries, "rails_minify/async_queries"
+  autoload :Doctor, "rails_minify/doctor"
+  autoload :SqlProfile, "rails_minify/sql_profile"
+
+  # The JITs already on when this file is required (Bundler.require, before
+  # Rails configures anything): turned on from the command line or the
+  # environment, so they compile boot too. minify:doctor reports them.
+  JIT_AT_BOOT = %i[yjit zjit].select { |jit| RubyVM.const_defined?(jit.upcase) && RubyVM.const_get(jit.upcase).enabled? }.freeze
+
   module Patches
     autoload :ActionTextPlainText, "rails_minify/patches/action_text_plain_text"
     autoload :OmniAuthPassthrough, "rails_minify/patches/omniauth_passthrough"

metadata

@@ -1,11 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: rails-minify
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Sampo Kuokkanen
-bindir: bin
+bindir: exe
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: activesupport-minimizer
   requirement: !ruby/object:Gem::Requirement
@@ -37,24 +37,30 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.2'
-description: 'One switch for what held up in production: Active Record strict mode
-  (only readers and writers per column), enum methods without closures and ZJIT turned
-  on after boot. minify:check finds callback problems. Experimental: patches for parts
-  of Rails and common gems that redo work on every request, and the other minimizer
+description: 'Active Record strict mode (only readers and writers per column), minify:check
+  for callback problems, minify:doctor for production configuration that costs speed
+  or memory, and SQL profiling from production traffic. Experimental: patches for
+  parts of Rails and common gems that redo work on every request, and the other minimizer
   parts.'
-executables: []
+executables:
+- rails-minify-ab
 extensions: []
 extra_rdoc_files: []
 files:
 - CHANGELOG.md
 - LICENSE.txt
 - README.md
+- exe/rails-minify-ab
 - lib/rails-minify.rb
 - lib/rails_minify.rb
+- lib/rails_minify/ab.rb
+- lib/rails_minify/async_queries.rb
+- lib/rails_minify/doctor.rb
 - lib/rails_minify/patches/action_text_plain_text.rb
 - lib/rails_minify/patches/omniauth_passthrough.rb
 - lib/rails_minify/patches/request_host.rb
 - lib/rails_minify/railtie.rb
+- lib/rails_minify/sql_profile.rb
 - lib/rails_minify/version.rb
 homepage: https://github.com/sampokuokkanen/rails-minify/tree/main/rails-minify
 licenses:
@@ -80,6 +86,6 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 rubygems_version: 4.0.20
 specification_version: 4
-summary: Strict Active Record attribute methods, ZJIT after boot, and checks for callback
-  problems
+summary: Strict Active Record attribute methods, production profiling, and checks
+  for callback problems
 test_files: []