prosopite 2.1.2 → 2.5.0

data/lib/generators/prosopite/install_generator.rb

@@ -0,0 +1,53 @@
+require 'rails/generators'
+
+module Prosopite
+  module Generators
+    # rails generate prosopite:install
+    class InstallGenerator < Rails::Generators::Base
+      source_root File.expand_path('templates', __dir__)
+
+      desc 'Creates config/initializers/prosopite.rb and scans every test for N+1 queries'
+
+      def create_initializer
+        template 'prosopite.rb', 'config/initializers/prosopite.rb'
+      end
+
+      def scan_tests
+        spec_helper = %w[spec/rails_helper.rb spec/spec_helper.rb].find { |path| File.exist?(File.join(destination_root, path)) }
+
+        if spec_helper
+          configure = File.read(File.join(destination_root, spec_helper))[/RSpec\.configure do \|(\w+)\|\r?\n/]
+          return say("Scan each spec in #{spec_helper}, see https://github.com/charkost/prosopite#tests") unless configure
+
+          config = configure[/\|(\w+)\|/, 1]
+          inject_into_file spec_helper, after: configure do
+            "  #{config}.before { |example| Prosopite.scan(example.location) unless example.metadata[:prosopite] == false }\n" \
+              "  #{config}.after { Prosopite.finish }\n"
+          end
+        elsif File.exist?(File.join(destination_root, 'test/test_helper.rb'))
+          append_to_file 'test/test_helper.rb', <<~RUBY
+
+            class ActiveSupport::TestCase
+              setup { Prosopite.scan("\#{self.class}#\#{name}") }
+              teardown { Prosopite.finish }
+            end
+          RUBY
+        end
+      end
+
+      def mention_pg_query
+        database = File.join(destination_root, 'config/database.yml')
+        return unless File.exist?(database) && File.read(database).include?('postgresql')
+
+        say "With PostgreSQL, gem 'pg_query' makes the fingerprints of queries more precise, if it installs on your systems"
+      end
+
+      private
+
+      def factory_bot?
+        gemfile = File.join(destination_root, 'Gemfile')
+        File.exist?(gemfile) && File.read(gemfile).match?(/['"]factory_bot(_rails)?['"]/)
+      end
+    end
+  end
+end

data/lib/generators/prosopite/templates/prosopite.rb.tt

@@ -0,0 +1,18 @@
+# Finds N+1 queries in development and in tests, see https://github.com/charkost/prosopite
+if Rails.env.development?
+  # In tests, the scan of each test covers its requests
+  require 'prosopite/middleware/rack'
+  Rails.configuration.middleware.use(Prosopite::Middleware::Rack)
+end
+
+Rails.application.config.after_initialize do
+  if Rails.env.development?
+    Prosopite.add_footer = true
+  elsif Rails.env.test?
+    Prosopite.raise = true
+<% if factory_bot? -%>
+    # Leave out the N+1 queries of factories, which only set up the tests
+    Prosopite.pause_during('factory_bot.run_factory')
+<% end -%>
+  end
+end

data/lib/prosopite/fixes.rb

@@ -0,0 +1,181 @@
+module Prosopite
+  # Suggests fixes for an N+1 from the associations whose loading runs queries like its
+  # own: on the table of the associated model, with a single condition on the foreign key
+  # of the association, or on the primary key for belongs_to. Other queries, like ones
+  # with more conditions, aggregates or joins, run whether an association is loaded or not.
+  module Fixes
+    # Comments around the query, e.g. of query_log_tags
+    LEADING_COMMENTS = %r{\A\s*(?:/\*.*?\*/\s*)+}m
+    TRAILING_COMMENTS = %r{\s*(?:/\*.*?\*/\s*)+\z}m
+    # A query that loads records, and the table it loads them from
+    LOAD = %r{\A\s*(?:/\*.*?\*/\s*)*SELECT\s+(?:DISTINCT\s+)?(?:[`"]?\w+[`"]?\.)?\*\s+FROM\s+[`"]?(\w+)[`"]?(?:\s|\z)}im
+    QUERY = /\A
+      SELECT\s+(?<select>.+?)\s+
+      FROM\s+(?<table>[`"]?\w+[`"]?)\s+
+      WHERE\s+(?<column>(?:[`"]?\w+[`"]?\.)?[`"]?\w+[`"]?)\s*
+      (?:=\s*(?:\$\d+|\?|-?\d+|'[^']*'|:\w+)|IN\s*\([^()]*\))
+      (?:\s+ORDER\s+BY\s+[^;]+?)?
+      (?:\s+LIMIT\s+\S+)?
+      (?:\s+OFFSET\s+\S+)?
+      \s*;?\s*
+    \z/xim
+    COLUMN = /(?:[`"]?\w+[`"]?\.)?[`"]?\w+[`"]?/
+    # Associations that can't be told apart without where their records were loaded
+    MAX_AMBIGUOUS = 3
+
+    class << self
+      # The block returns [order, location] for where records of a model were loaded, with
+      # order telling how recent that was, to pick the associations of the records looped on.
+      # Without such records, loop_table is the table of the last records loaded before the
+      # N+1, and when no association is of it, the N+1 loops on records that no suggestion
+      # fits, e.g. through a polymorphic association.
+      def for(query, index = self.index, loop_table: nil)
+        match = parse(query)
+        kind = match && kind(match[:select])
+        return [] unless kind
+
+        reflections = associations(query, index).select { |reflection| kind == :load || reflection.collection? }
+        loads = reflections.to_h { |reflection| [reflection, block_given? ? yield(reflection.active_record) : nil] }
+        latest = loads.values.compact.map(&:first).max
+
+        if latest
+          reflections = reflections.select { |reflection| loads[reflection]&.first == latest }
+        elsif reflections.size > MAX_AMBIGUOUS ||
+              (loop_table && reflections.none? { |reflection| reflection.active_record.table_name == loop_table })
+          return []
+        end
+
+        reflections.map { |reflection| suggestion(reflection, kind, loads[reflection]&.last) }
+      end
+
+      def associations(query, index = self.index)
+        match = parse(query)
+        return [] unless match
+
+        table = unquote(match[:table])
+        qualifier, column = match[:column].split('.', 2).map { |name| unquote(name) }
+        qualifier, column = nil, qualifier unless column
+        return [] if qualifier && qualifier != table
+
+        index.fetch([table, column], [])
+      end
+
+      # The associations by the table and the column of their queries, built again
+      # only when the models change, e.g. when reloading code in development
+      def index
+        models = ActiveRecord::Base.descendants
+        models_key = models.map(&:object_id)
+        return @index if @index_models_key == models_key
+
+        index = {}
+        models.flat_map(&:reflect_on_all_associations).uniq.each do |reflection|
+          key = query_key(reflection)
+          (index[key] ||= []) << reflection if key
+        end
+        @index = index
+        @index_models_key = models_key
+        index
+      end
+
+      private
+
+      def parse(query)
+        query.sub(LEADING_COMMENTS, '').sub(TRAILING_COMMENTS, '').match(QUERY)
+      end
+
+      def kind(select)
+        if select.match?(/\A(?:[`"]?\w+[`"]?\.)?\*\z/)
+          :load
+        elsif select.match?(/\ACOUNT\(/i)
+          :count
+        elsif select.match?(/\A1 AS one\z/i)
+          :exists
+        elsif select.match?(/\A#{COLUMN}(?:\s*,\s*#{COLUMN})*\z/)
+          # pluck, which uses the records of a loaded association
+          :load
+        end
+      end
+
+      def suggestion(reflection, kind, location)
+        owner = reflection.active_record
+        fix = "#{owner.name}.includes(:#{reflection.name})"
+        fix += " where the #{owner.table_name} are loaded, at #{location}" if location
+
+        case kind
+        when :count then "#{fix}, then #{reflection.name}.size; or a counter cache"
+        when :exists then "#{fix}, then #{reflection.name}.any?"
+        else fix
+        end
+      end
+
+      def unquote(name)
+        name.delete('`"')
+      end
+
+      def query_key(reflection)
+        owner = reflection.active_record.name
+        # Through associations join, Rails defines HABTM_ models for its join tables, and
+        # scopes and subclasses of single table inheritance add conditions to the queries
+        return if reflection.through_reflection? || reflection.polymorphic? || owner.nil? || owner.include?('HABTM_')
+        return if reflection.scope || !reflection.klass.descends_from_active_record?
+
+        column = reflection.belongs_to? ? reflection.association_primary_key : reflection.foreign_key
+        [reflection.klass.table_name, column.to_s]
+      rescue StandardError
+        # An association to a class that doesn't exist, isn't a model, or has no database
+        nil
+      end
+    end
+
+    # Where the records that N+1s loop on were loaded: the last query on their table before
+    # each N+1, with its order among the queries of the scan
+    class Loads
+      def initialize(state, backtrace_cleaner)
+        @holder = state[:prosopite_query_holder]
+        @callers = state[:prosopite_query_caller]
+        @backtrace_cleaner = backtrace_cleaner
+        # In the order of their first query. A copy, as an ActionController::Live action may
+        # still be adding to them.
+        @keys = @holder.keys
+        @positions = @keys.each_with_index.to_h
+        @found = {}
+      end
+
+      def at(location_key, table)
+        @found.fetch([location_key, table]) { |key| @found[key] = find(location_key, table) }
+      end
+
+      # The table of the last records loaded by a query that ran once before the N+1,
+      # which likely loaded the records it loops on
+      def loop_table(location_key)
+        before(location_key).each do |position|
+          queries = @holder[@keys[position]]
+          table = queries.first[LOAD, 1] if queries.size == 1
+          return table if table
+        end
+        nil
+      end
+
+      private
+
+      # The positions of the queries before the N+1, the latest first
+      def before(location_key)
+        (@positions.fetch(location_key, @keys.size) - 1).downto(0)
+      end
+
+      def find(location_key, table)
+        before(location_key).each do |position|
+          key = @keys[position]
+          next unless @holder[key].first[LOAD, 1] == table
+
+          # The call stacks of single queries are kept up to a limit
+          caller = @callers[key]
+          frame = caller && @backtrace_cleaner.clean(caller.map(&:to_s)).first
+          return [position, frame&.sub(/:in\b.*\z/, '')]
+        end
+
+        nil
+      end
+    end
+  end
+end

data/lib/prosopite/middleware/rack.rb

@@ -1,16 +1,226 @@
+require 'json'
+
 module Prosopite
   module Middleware
     class Rack
+      # Carries the N+1 queries of responses that aren't pages, e.g. of fetch requests
+      HEADER = 'x-prosopite-notifications'
+      # Proxies and servers limit the size of all headers, e.g. nginx to 4 KB by default
+      HEADER_LIMIT = 2048
+      HEADER_MESSAGE_LIMIT = 400
+
+      FOOTER_STYLE = <<~CSS.freeze
+        #prosopite-footer { position: fixed; left: 0; bottom: 0; z-index: 2147483647; box-sizing: border-box;
+          max-width: 100vw; max-height: 50vh; overflow: auto; margin: 0; padding: 4px 8px; text-align: left;
+          background: #fff5f5; color: #a40000; border: 1px solid #a40000; border-width: 1px 1px 0 0;
+          border-top-right-radius: 6px; font: 12px/1.4 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; }
+        #prosopite-footer summary { cursor: pointer; font-weight: bold; }
+        #prosopite-footer pre { margin: 4px 0 0; white-space: pre-wrap; font: inherit; color: #222; }
+      CSS
+
+      # Shows the N+1 queries of the fetch and XHR requests of a page, like the ones of
+      # Turbo, from their header. Turbo runs it again on every visit, so it runs once.
+      REQUESTS_SCRIPT = <<~JS.freeze
+        (function (showFooter, showConsole) {
+          if (window.prosopite) return;
+          window.prosopite = true;
+
+          function show(value) {
+            var data;
+            try { data = JSON.parse(value); } catch (e) { return; }
+
+            if (showConsole) data.messages.forEach(function (message) { console.warn(message); });
+            if (!showFooter) return;
+
+            var footer = document.getElementById('prosopite-footer');
+            if (!footer) {
+              footer = document.createElement('details');
+              footer.id = 'prosopite-footer';
+              footer.appendChild(document.createElement('summary'));
+              footer.appendChild(document.createElement('pre'));
+              document.body.appendChild(footer);
+            }
+            var count = Number(footer.getAttribute('data-count') || 0) + data.count;
+            footer.setAttribute('data-count', count);
+            footer.querySelector('summary').textContent =
+              'Prosopite: ' + count + ' N+1 ' + (count === 1 ? 'query' : 'queries') + ' detected';
+            footer.querySelector('pre').textContent +=
+              data.messages.join('') + (data.count > data.messages.length ? '...\\n\\n' : '');
+          }
+
+          var fetch = window.fetch;
+          if (fetch) {
+            window.fetch = function () {
+              return fetch.apply(window, arguments).then(function (response) {
+                var value = response.headers.get('#{HEADER}');
+                if (value) show(value);
+                return response;
+              });
+            };
+          }
+
+          var open = XMLHttpRequest.prototype.open;
+          XMLHttpRequest.prototype.open = function () {
+            if (!this.prosopite) {
+              this.prosopite = true;
+              this.addEventListener('load', function () {
+                // Reading a header that a cross-origin response doesn't expose logs an error
+                if (!/^#{HEADER}:/im.test(this.getAllResponseHeaders())) return;
+                show(this.getResponseHeader('#{HEADER}'));
+              });
+            }
+            return open.apply(this, arguments);
+          };
+        })
+      JS
+
       def initialize(app)
         @app = app
       end
 
       def call(env)
-        Prosopite.scan 
-        @app.call(env)
+        return @app.call(env) if Prosopite.disabled?
+
+        req = ::Rack::Request.new(env)
+        Prosopite.scan("#{req.request_method} #{req.path}")
+        status, headers, body = @app.call(env)
+        notifications = Prosopite.finish || []
+
+        if Prosopite.add_footer || Prosopite.browser_console
+          body, page = add_to_page(env, headers, body, notifications)
+          headers[HEADER] = header_value(notifications) if !page && notifications.any?
+        end
+
+        [status, headers, body]
       ensure
         Prosopite.finish
       end
+
+      private
+
+      # Adds the N+1 queries and the script for later requests to a page. Responses that
+      # aren't pages, like the ones of Turbo frames and streams, return false.
+      def add_to_page(env, headers, body, notifications)
+        return [body, false] if env['HTTP_TURBO_FRAME'] || !injectable?(headers, body)
+
+        parts = body.to_ary
+        return [body, false] unless parts
+
+        html = String.new(encoding: Encoding::BINARY)
+        parts.each { |part| html << part.b }
+
+        index = html.rindex(%r{</body>}i)
+        return [parts, false] unless index
+
+        html.insert(index, page_html(env, headers, notifications).b)
+        # In the encoding of the page again, e.g. for middleware that add to it after this one
+        html.force_encoding(parts.first.encoding)
+        html.force_encoding(Encoding::BINARY) unless html.valid_encoding?
+
+        content_length = headers.keys.find { |k| k.casecmp?('content-length') }
+        headers[content_length] = html.bytesize.to_s if content_length
+
+        [[html], true]
+      end
+
+      # Only buffered bodies can be changed: Rack allows middleware to read a body
+      # through to_ary, but not to iterate it, which would break streaming.
+      def injectable?(headers, body)
+        body.respond_to?(:to_ary) &&
+          header(headers, 'content-type').to_s.include?('text/html') &&
+          !header(headers, 'content-disposition').to_s.include?('attachment') &&
+          header(headers, 'content-encoding').nil? &&
+          header(headers, 'transfer-encoding').nil?
+      end
+
+      def header(headers, name)
+        key = headers.keys.find { |k| k.casecmp?(name) }
+        headers[key] if key
+      end
+
+      def page_html(env, headers, notifications)
+        nonce = content_security_policy_nonce(env)
+        nonce_attr = nonce ? %( nonce="#{::Rack::Utils.escape_html(nonce)}") : ''
+        policy = content_security_policy(env, headers)
+        messages = notifications.map { |notification| Prosopite.notification_message(notification) }
+
+        html = String.new('')
+        if Prosopite.add_footer
+          html << "<style#{nonce_attr}>#{FOOTER_STYLE}</style>" if inline_allowed?(policy, 'style-src', nonce)
+          html << footer_html(messages) if messages.any?
+        end
+
+        # A blocked script would also be reported
+        return html unless inline_allowed?(policy, 'script-src', nonce)
+
+        script = String.new('')
+        script << messages.map { |message| "console.warn(#{js_string(message)});" }.join if Prosopite.browser_console
+        script << "#{REQUESTS_SCRIPT}(#{!!Prosopite.add_footer}, #{!!Prosopite.browser_console});"
+        html << "<script#{nonce_attr}>#{script}</script>"
+      end
+
+      # The Content Security Policy that the response will have. Rails adds its own outside
+      # this middleware, unless the response has one, so it's built the same way here.
+      def content_security_policy(env, headers)
+        policy = header(headers, 'content-security-policy')
+        rails_policy = env['action_dispatch.content_security_policy']
+        return policy if policy || rails_policy.nil? || header(headers, 'content-security-policy-report-only')
+
+        request = ::ActionDispatch::Request.new(env)
+        return if request.content_security_policy_report_only
+
+        rails_policy.build(request.controller_instance || request, request.content_security_policy_nonce,
+                           request.content_security_policy_nonce_directives)
+      end
+
+      # Whether the policy lets an inline element of the directive run, with the nonce if any.
+      # Browsers ignore 'unsafe-inline' in directives with nonces, hashes or 'strict-dynamic'.
+      def inline_allowed?(policy, directive, nonce)
+        return true unless policy
+
+        directives = policy.split(';').map(&:split)
+        sources = directives.assoc("#{directive}-elem") || directives.assoc(directive) || directives.assoc('default-src')
+        return true unless sources
+        return true if nonce && sources.include?("'nonce-#{nonce}'")
+
+        sources.include?("'unsafe-inline'") && sources.none? { |source| source.match?(/\A'(?:nonce-|sha\d+-|strict-dynamic')/) }
+      end
+
+      def footer_html(messages)
+        summary = "Prosopite: #{messages.size} N+1 #{messages.size == 1 ? 'query' : 'queries'} detected"
+
+        "<details id=\"prosopite-footer\" data-count=\"#{messages.size}\"><summary>#{summary}</summary>" \
+          "<pre>#{::Rack::Utils.escape_html(messages.join)}</pre></details>"
+      end
+
+      # As many messages as fit, as ASCII JSON, with the count of all the N+1 queries
+      def header_value(notifications)
+        messages = []
+        notifications.each do |notification|
+          message = Prosopite.notification_message(notification)
+          message = "#{message[0, HEADER_MESSAGE_LIMIT]}...\n\n" if message.size > HEADER_MESSAGE_LIMIT
+          break if header_json(notifications.size, messages + [message]).bytesize > HEADER_LIMIT
+
+          messages << message
+        end
+
+        header_json(notifications.size, messages)
+      end
+
+      def header_json(count, messages)
+        JSON.generate({ count: count, messages: messages }, ascii_only: true)
+      end
+
+      # JSON, with the characters that could close the script tag escaped
+      def js_string(str)
+        JSON.generate(str).gsub(/[<>&

]/) { |c| format('\\u%04x', c.ord) }
+      end
+
+      def content_security_policy_nonce(env)
+        return unless env['action_dispatch.content_security_policy_nonce_generator']
+
+        ::ActionDispatch::Request.new(env).content_security_policy_nonce
+      end
     end
   end
 end

data/lib/prosopite/middleware/sidekiq.rb

@@ -3,8 +3,8 @@ module Prosopite
     class Sidekiq
       include ::Sidekiq::ServerMiddleware
   
-      def call(_worker, _msg, _queue)
-        Prosopite.scan
+      def call(worker, msg, queue)
+        Prosopite.scan("#{worker.class.name} JID-#{msg['jid']} queue=#{queue}")
         yield
       ensure
         Prosopite.finish

data/lib/prosopite/minitest.rb

@@ -0,0 +1,23 @@
+require 'minitest'
+require 'prosopite'
+
+module Prosopite
+  # Assertions for Minitest, included in Minitest::Test
+  module Assertions
+    def assert_no_n_plus_one_queries(&block)
+      n_plus_one_queries = Prosopite.n_plus_one_queries(&block)
+      message = lambda do
+        "Expected no N+1 queries, but there were:\n\n" +
+          n_plus_one_queries.map { |notification| Prosopite.notification_message(notification) }.join
+      end
+
+      assert n_plus_one_queries.empty?, message
+    end
+
+    def assert_n_plus_one_queries(&block)
+      refute Prosopite.n_plus_one_queries(&block).empty?, 'Expected N+1 queries, but there were none'
+    end
+  end
+end
+
+Minitest::Test.include(Prosopite::Assertions)

data/lib/prosopite/rspec.rb

@@ -0,0 +1,20 @@
+require 'prosopite'
+
+# expect { ... }.not_to have_n_plus_one_queries
+RSpec::Matchers.define :have_n_plus_one_queries do
+  supports_block_expectations
+
+  match do |block|
+    @n_plus_one_queries = Prosopite.n_plus_one_queries(&block)
+    @n_plus_one_queries.any?
+  end
+
+  failure_message do
+    'expected N+1 queries, but there were none'
+  end
+
+  failure_message_when_negated do
+    "expected no N+1 queries, but there were:\n\n" +
+      @n_plus_one_queries.map { |notification| Prosopite.notification_message(notification) }.join
+  end
+end

data/lib/prosopite/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Prosopite
-  VERSION = "2.1.2"
+  VERSION = "2.5.0"
 end