prosopite 1.0.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3df6a0299972911e48ea44330995864e2cd2191ec66c199296ba324a5fa8e639
-  data.tar.gz: 4f9ee347a8cf52fd742ee66586c7e4b601c8436a4bcabe1b1afdef5e09293f44
+  metadata.gz: 71358661958f67bb09075fa89db0f0c09868c0a53780afa3780a8cc7d4ded49a
+  data.tar.gz: 1de6a0ab099524457cf49c60d1f5ae0041dffc1b9e281202ccea77cb2cd89ba9
 SHA512:
-  metadata.gz: 668bc7e58bcabd77231adc831054d56ca572e1e9857f87736d1fd06048c1eeaac9c0c0ab970c6ed6f8ce70670ae51277c3c4f56012d0ec61240c0e728c1f3795
-  data.tar.gz: '0341920f4f1beddc2b9c46553f9284724ecddfe50f30579a7cc9627c93fdf2d3c714bb57fd453dd7a4783a86951b9dd2cb0a79e721189bce3b89eda0c119c8bf'
+  metadata.gz: 8478248b48b55a59eedfde1c472382eccc543a0f2ce03a891174c326d03a17db19cd3c9a393a2f1a89568a2c275e04ae8ca71d13f30cf67fef64870ed5531494
+  data.tar.gz: de26d196d0a767511b7a9e1f6cf5c87b00c838e17dc2e630f3689a0d43b7a47542c5c0e28f70f3b49a9dca7b577349c6dc66c97b14535cd9e010f83be9f2c58c

data/.github/workflows/ci.yml

@@ -4,7 +4,7 @@ jobs:
   test:
     strategy:
       matrix:
-        ruby: [2.5, 2.6, 2.7, '3.0', head]
+        ruby: [2.7, '3.0', 3.1, head]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2

data/Gemfile.lock

@@ -1,11 +1,24 @@
 PATH
   remote: .
   specs:
-    prosopite (0.2.1)
+    prosopite (1.3.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    actionpack (6.1.3)
+      actionview (= 6.1.3)
+      activesupport (= 6.1.3)
+      rack (~> 2.0, >= 2.0.9)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.2.0)
+    actionview (6.1.3)
+      activesupport (= 6.1.3)
+      builder (~> 3.1)
+      erubi (~> 1.4)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.1, >= 1.2.0)
     activemodel (6.1.3)
       activesupport (= 6.1.3)
     activerecord (6.1.3)
@@ -17,20 +30,55 @@ GEM
       minitest (>= 5.1)
       tzinfo (~> 2.0)
       zeitwerk (~> 2.3)
+    ansi (1.5.0)
+    builder (3.2.4)
     coderay (1.1.3)
     concurrent-ruby (1.1.8)
+    crass (1.0.6)
+    erubi (1.12.0)
     factory_bot (6.1.0)
       activesupport (>= 5.0.0)
     i18n (1.8.9)
       concurrent-ruby (~> 1.0)
+    loofah (2.19.1)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.5.9)
     method_source (1.0.0)
+    mini_portile2 (2.8.1)
     minitest (5.14.3)
+    minitest-reporters (1.5.0)
+      ansi
+      builder
+      minitest (>= 5.0)
+      ruby-progressbar
+    nokogiri (1.14.1)
+      mini_portile2 (~> 2.8.0)
+      racc (~> 1.4)
+    nokogiri (1.14.1-x86_64-linux)
+      racc (~> 1.4)
     pg_query (1.3.0)
     pry (0.14.0)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    rake (13.0.3)
+    racc (1.6.2)
+    rack (2.2.6.2)
+    rack-test (2.0.2)
+      rack (>= 1.3)
+    rails-dom-testing (2.0.3)
+      activesupport (>= 4.2.0)
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.5.0)
+      loofah (~> 2.19, >= 2.19.1)
+    railties (6.1.3)
+      actionpack (= 6.1.3)
+      activesupport (= 6.1.3)
+      method_source
+      rake (>= 0.8.7)
+      thor (~> 1.0)
+    rake (13.0.6)
+    ruby-progressbar (1.11.0)
     sqlite3 (1.4.2)
+    thor (1.2.1)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
     zeitwerk (2.4.2)
@@ -43,9 +91,11 @@ DEPENDENCIES
   activerecord
   factory_bot
   minitest
+  minitest-reporters
   pg_query
   prosopite!
   pry
+  railties
   rake (~> 13.0)
   sqlite3
 

data/README.md

@@ -115,10 +115,36 @@ Or install it yourself as:
 
 The preferred type of notifications can be configured with:
 
+* `Prosopite.min_n_queries`: Minimum number of N queries to report per N+1 case. Defaults to 2.
+* `Prosopite.raise = true`: Raise warnings as exceptions
 * `Prosopite.rails_logger = true`: Send warnings to the Rails log
 * `Prosopite.prosopite_logger = true`: Send warnings to `log/prosopite.log`
 * `Prosopite.stderr_logger = true`: Send warnings to STDERR
-* `Prosopite.raise = true`: Raise warnings as exceptions
+* `Prosopite.custom_logger = my_custom_logger`:
+* `Prosopite.backtrace_cleaner = my_custom_backtrace_cleaner`: use a different [ActiveSupport::BacktraceCleaner](https://api.rubyonrails.org/classes/ActiveSupport/BacktraceCleaner.html). Default to `Rails.backtrace_cleaner` if present.
+
+### Custom Logging Configuration
+
+You can supply a custom logger with the `Prosopite.custom_logger` setting.
+
+This is useful for circumstances where you don't want your logs to be
+highlighted with red, or you want logs sent to a custom location.
+
+One common scenario is that you may be generating json logs and sending them to
+Datadog, ELK stack, or similar, and don't want to have to remove the default red
+escaping data from messages sent to the Rails logger, or want to tag them
+differently with your own custom logger.
+
+```ruby
+# Turns off logging with red highlights, but still sends them to the Rails logger
+Prosopite.custom_logger = Rails.logger
+```
+
+```ruby
+# Use a completely custom logging instance
+Prosopite.custom_logger = MyLoggerClass.new
+
+```
 
 ## Development Environment Usage
 
@@ -127,11 +153,12 @@ Prosopite auto-detection can be enabled on all controllers:
 ```ruby
 class ApplicationController < ActionController::Base
   unless Rails.env.production?
-    before_action do
-      Prosopite.scan
-    end
+    around_action :n_plus_one_detection
 
-    after_action do
+    def n_plus_one_detection
+      Prosopite.scan
+      yield
+    ensure
       Prosopite.finish
     end
   end
@@ -178,10 +205,16 @@ WARNING: scan/finish should run before/after **each** test and NOT before/after
 
 ## Allow list
 
-Ignore notifications for call stacks containing one or more substrings:
+Ignore notifications for call stacks containing one or more substrings / regex:
+
+```ruby
+Prosopite.allow_stack_paths = ['substring_in_call_stack', /regex/]
+```
+
+Ignore notifications matching a specific SQL query:
 
 ```ruby
-Prosopite.allow_list = ['substring_in_call_stack']
+Prosopite.ignore_queries = [/regex_match/, "SELECT * from EXACT_STRING_MATCH"]
 ```
 
 ## Scanning code outside controllers or tests
@@ -194,6 +227,55 @@ Prosopite.scan
 Prosopite.finish
 ```
 
+In block form the `Prosopite.finish` is called automatically for you at the end of the block:
+
+```ruby
+Prosopite.scan do
+  <code to scan>
+end
+```
+
+The result of the code block is also returned by `Prosopite.scan`, so you can wrap calls as follows:
+
+```ruby
+my_object = Prosopite.scan do
+  MyObjectFactory.create(params)
+end
+```
+
+## Pausing and resuming scans
+
+Scans can be paused:
+
+```ruby
+Prosopite.scan
+# <code to scan>
+Prosopite.pause
+# <code that has n+1s>
+Prosopite.resume
+# <code to scan>
+Prosopite.finish
+```
+
+You can also pause items in a block, and the `Prosopite.resume` will be done
+for you automatically:
+
+```ruby
+Prosopite.scan
+# <code to scan>
+
+result = Prosopite.pause do
+  # <code that has n+1s>
+end
+
+Prosopite.finish
+```
+
+Pauses can be ignored with `Prosopite.ignore_pauses = true` in case you want to remember their N+1 queries.
+
+An example of when you might use this is if you are [testing Active Jobs inline](https://guides.rubyonrails.org/testing.html#testing-jobs),
+and don't want to run Prosopite on background job code, just foreground app code. In that case you could write an [Active Job callback](https://edgeguides.rubyonrails.org/active_job_basics.html#callbacks) that pauses the scan while the job is running.
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/charkost/prosopite.

data/lib/prosopite/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Prosopite
-  VERSION = "1.0.0"
+  VERSION = "1.3.0"
 end

data/lib/prosopite.rb

@@ -1,12 +1,29 @@
 
 module Prosopite
+  DEFAULT_ALLOW_LIST = %w(active_record/associations/preloader active_record/validations/uniqueness)
+
   class NPlusOneQueriesError < StandardError; end
   class << self
     attr_writer :raise,
                 :stderr_logger,
                 :rails_logger,
                 :prosopite_logger,
-                :allow_list
+                :custom_logger,
+                :allow_stack_paths,
+                :ignore_queries,
+                :ignore_pauses,
+                :min_n_queries,
+                :backtrace_cleaner
+
+    def allow_list=(value)
+      puts "Prosopite.allow_list= is deprecated. Use Prosopite.allow_stack_paths= instead."
+
+      self.allow_stack_paths = value
+    end
+
+    def backtrace_cleaner
+      @backtrace_cleaner ||= Rails.backtrace_cleaner
+    end
 
     def scan
       tc[:prosopite_scan] ||= false
@@ -18,17 +35,52 @@ module Prosopite
       tc[:prosopite_query_holder] = Hash.new { |h, k| h[k] = [] }
       tc[:prosopite_query_caller] = {}
 
-      @allow_list ||= []
+      @allow_stack_paths ||= []
+      @ignore_pauses ||= false
+      @min_n_queries ||= 2
 
       tc[:prosopite_scan] = true
+
+      if block_given?
+        begin
+          block_result = yield
+          finish
+          block_result
+        ensure
+          tc[:prosopite_scan] = false
+        end
+      end
     end
 
     def tc
       Thread.current
     end
 
+    def pause
+      if @ignore_pauses
+        return block_given? ? yield : nil
+      end
+
+      if block_given?
+        begin
+          previous = tc[:prosopite_scan]
+          tc[:prosopite_scan] = false
+          yield
+        ensure
+          tc[:prosopite_scan] = previous
+        end
+      else
+        tc[:prosopite_scan] = false
+      end
+    end
+
+    def resume
+      tc[:prosopite_scan] = true
+    end
+
     def scan?
-      tc[:prosopite_scan]
+      !!(tc[:prosopite_scan] && tc[:prosopite_query_counter] &&
+         tc[:prosopite_query_holder] && tc[:prosopite_query_caller])
     end
 
     def finish
@@ -38,14 +90,18 @@ module Prosopite
 
       create_notifications
       send_notifications if tc[:prosopite_notifications].present?
+
+      tc[:prosopite_query_counter] = nil
+      tc[:prosopite_query_holder] = nil
+      tc[:prosopite_query_caller] = nil
     end
 
     def create_notifications
       tc[:prosopite_notifications] = {}
 
       tc[:prosopite_query_counter].each do |location_key, count|
-        if count > 1
-          fingerprints = tc[:prosopite_query_holder][location_key].map do |q|
+        if count >= @min_n_queries
+          fingerprints = tc[:prosopite_query_holder][location_key].group_by do |q|
             begin
               fingerprint(q)
             rescue
@@ -53,13 +109,17 @@ module Prosopite
             end
           end
 
-          kaller = tc[:prosopite_query_caller][location_key]
+          queries = fingerprints.values.select { |q| q.size >= @min_n_queries }
 
-          if fingerprints.uniq.size == 1 && !kaller.any? { |f| @allow_list.any? { |s| f.include?(s) } }
-            queries = tc[:prosopite_query_holder][location_key]
+          next unless queries.any?
 
-            unless kaller.any? { |f| f.include?('active_record/validations/uniqueness') }
-              tc[:prosopite_notifications][queries] = kaller
+          kaller = tc[:prosopite_query_caller][location_key]
+          allow_list = (@allow_stack_paths + DEFAULT_ALLOW_LIST)
+          is_allowed = kaller.any? { |f| allow_list.any? { |s| f.match?(s) } }
+
+          unless is_allowed
+            queries.each do |q|
+              tc[:prosopite_notifications][q] = kaller
             end
           end
         end
@@ -105,7 +165,7 @@ module Prosopite
 
       query.gsub!(/\btrue\b|\bfalse\b/i, "?")
 
-      query.gsub!(/[0-9+-][0-9a-f.xb+-]*/, "?")
+      query.gsub!(/[0-9+-][0-9a-f.x+-]*/, "?")
       query.gsub!(/[xb.+-]\?/, "?")
 
       query.strip!
@@ -128,6 +188,7 @@ module Prosopite
     end
 
     def send_notifications
+      @custom_logger ||= false
       @rails_logger ||= false
       @stderr_logger ||= false
       @prosopite_logger ||= false
@@ -137,14 +198,20 @@ module Prosopite
 
       tc[:prosopite_notifications].each do |queries, kaller|
         notifications_str << "N+1 queries detected:\n"
+
         queries.each { |q| notifications_str << "  #{q}\n" }
+
         notifications_str << "Call stack:\n"
+        kaller = backtrace_cleaner.clean(kaller)
         kaller.each do |f|
-          notifications_str << "  #{f}\n" unless f.include?(Bundler.bundle_path.to_s)
+          notifications_str << "  #{f}\n"
         end
+
         notifications_str << "\n"
       end
 
+      @custom_logger.warn(notifications_str) if @custom_logger
+
       Rails.logger.warn(red(notifications_str)) if @rails_logger
       $stderr.puts(red(notifications_str)) if @stderr_logger
 
@@ -161,14 +228,19 @@ module Prosopite
       str.split("\n").map { |line| "\e[91m#{line}\e[0m" }.join("\n")
     end
 
+    def ignore_query?(sql)
+      @ignore_queries ||= []
+      @ignore_queries.any? { |q| q === sql }
+    end
+
     def subscribe
       @subscribed ||= false
       return if @subscribed
 
       ActiveSupport::Notifications.subscribe 'sql.active_record' do |_, _, _, _, data|
-        sql = data[:sql]
+        sql, name = data[:sql], data[:name]
 
-        if scan? && sql.include?('SELECT') && data[:cached].nil?
+        if scan? && name != "SCHEMA" && sql.include?('SELECT') && data[:cached].nil? && !ignore_query?(sql)
           location_key = Digest::SHA1.hexdigest(caller.join)
 
           tc[:prosopite_query_counter][location_key] += 1

data/prosopite.gemspec

@@ -28,5 +28,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "minitest"
   spec.add_development_dependency "factory_bot"
   spec.add_development_dependency "activerecord"
+  spec.add_development_dependency "railties"
   spec.add_development_dependency "sqlite3"
+  spec.add_development_dependency "minitest-reporters"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prosopite
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Mpampis Kostas
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-17 00:00:00.000000000 Z
+date: 2023-02-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pry
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: railties
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest-reporters
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: N+1 auto-detection for Rails with zero false positives / false negatives
 email:
 - charkost.rb@gmail.com
@@ -118,8 +146,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6.2
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: N+1 auto-detection for Rails with zero false positives / false negatives