n_plus_one_control 0.3.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 292888473b7ae70cfdc9c8e9324170dfaf1d7048fc27a8c33a3ef09ee7b3fc3a
-  data.tar.gz: 99bf992bbe635524c8eaea3f6536f4813e05b4420bc1f26ecea8ab26fecfc0f1
+  metadata.gz: 48ad0338e6227c2b71a09254f3613db415d64fe5b40ee7df55ee1daddc46c0f5
+  data.tar.gz: c55895975614627ea2308dde866079bad496f5ddef81f295267f5b0d8b2670ac
 SHA512:
-  metadata.gz: cd45056252d7ae15236e8ae5d5880c0bda50f741ac3cf2079a42f5870cf378fb52d791b444c57c25218428d52e046841a5a762708a18ad868103d9e17f0d30f1
-  data.tar.gz: b17d7be7d184ffa37ce4f6162b694928eb96968cc5f4e05e42971d68954709ed33c43b36c5549d49823ac2578103d9420cf713dd4f6cbc12880e885203a87bb4
+  metadata.gz: b4f6b44d7ecccd6e3f53d65e4df916982b144b8381049ebec5659cdf5792cc9c1d1c7fdee22cb8610df155020220286cc2daf78a0dbe8b76fe6bceb778446c81
+  data.tar.gz: 63ac722a3b82c6c3263a79f0065f554352bf4ce2043bdec98f37139f8b21f441e9327bc575ff6e3bdde05e8eac95ce636615e74ed4e3415a0ae28312da43d2a7

data/CHANGELOG.md

@@ -0,0 +1,33 @@
+## 0.6.0 (2020-11-27)
+
+- Fix table stats summary when queries use backticks to surround table names ([@andrewhampton][])
+- Add support to test for linear query. ([@caalberts][])
+
+## 0.5.0 (2020-09-07)
+
+- **Ruby 2.5+ is required**. ([@palkan][])
+
+- Add support for multiple backtrace lines in verbose output. ([@palkan][])
+
+Could be specified via `NPLUSONE_BACKTRACE` env var.
+
+- Add `NPLUSONE_TRUNCATE` env var to truncate queries in verbose mode. ([@palkan][])
+
+- Support passing default filter via `NPLUSONE_FILTER` env var. ([@palkan][])
+
+- Add location tracing to SQLs in verbose mode. ([@palkan][])
+
+## 0.4.1 (2020-09-04)
+
+- Enhance failure message by showing differences in table hits. ([@palkan][])
+
+## 0.4.0 (2020-07-20)
+
+- Make scale factor available in tests via `#current_scale` method. ([@Earendil95][])
+
+- Start keeping a changelog. ([@palkan][])
+
+[@Earendil95]: https://github.com/Earendil95
+[@palkan]: https://github.com/palkan
+[@caalberts]: https://github.com/caalberts
+[@andrewhampton]: https://github.com/andrewhampton

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2017 palkan
+Copyright (c) 2017-2020 Vladimir Dementyev
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -1,4 +1,5 @@
-[![Gem Version](https://badge.fury.io/rb/n_plus_one_control.svg)](https://rubygems.org/gems/n_plus_one_control) [![Build Status](https://travis-ci.org/palkan/n_plus_one_control.svg?branch=master)](https://travis-ci.org/palkan/n_plus_one_control)
+[![Gem Version](https://badge.fury.io/rb/n_plus_one_control.svg)](https://rubygems.org/gems/n_plus_one_control)
+![Build](https://github.com/palkan/n_plus_one_control/workflows/Build/badge.svg)
 
 # N + 1 Control
 
@@ -31,7 +32,7 @@ Add this line to your application's Gemfile:
 
 ```ruby
 group :test do
-  gem 'n_plus_one_control'
+  gem "n_plus_one_control"
 end
 ```
 
@@ -47,8 +48,6 @@ First, add NPlusOneControl to your `spec_helper.rb`:
 
 ```ruby
 # spec_helper.rb
-...
-
 require "n_plus_one_control/rspec"
 ```
 
@@ -86,10 +85,45 @@ Availables modifiers:
 ```ruby
 # You can specify the RegExp to filter queries.
 # By default, it only considers SELECT queries.
-expect { ... }.to perform_constant_number_of_queries.matching(/INSERT/)
+expect { get :index }.to perform_constant_number_of_queries.matching(/INSERT/)
 
 # You can also provide custom scale factors
-expect { ... }.to perform_constant_number_of_queries.with_scale_factors(10, 100)
+expect { get :index }.to perform_constant_number_of_queries.with_scale_factors(10, 100)
+```
+
+#### Using scale factor in spec
+
+Let's suppose your action accepts parameter, which can make impact on the number of returned records:
+
+```ruby
+get :index, params: {per_page: 10}
+```
+
+Then it is enough to just change `per_page` parameter between executions and do not recreate records in DB. For this purpose, you can use `current_scale` method in your example:
+
+```ruby
+context "N+1", :n_plus_one do
+  before { create_list :post, 3 }
+
+  specify do
+    expect { get :index, params: {per_page: current_scale} }.to perform_constant_number_of_queries
+  end
+end
+```
+
+#### Other available matchers
+
+`perform_linear_number_of_queries(slope: 1)` allows you to test that a query generates linear number of queries with the given slope.  
+
+```ruby
+context "when has linear query", :n_plus_one do
+  populate { |n| create_list(:post, n) }
+
+  specify do
+    expect { Post.find_each { |p| p.user.name } }
+      .to perform_linear_number_of_queries(slope: 1)
+  end
+end
 ```
 
 ### Minitest
@@ -98,8 +132,6 @@ First, add NPlusOneControl to your `test_helper.rb`:
 
 ```ruby
 # test_helper.rb
-...
-
 require "n_plus_one_control/minitest"
 ```
 
@@ -115,6 +147,18 @@ def test_no_n_plus_one_error
 end
 ```
 
+You can also use `assert_perform_linear_number_of_queries` to test for linear queries:
+
+```ruby
+def test_no_n_plus_one_error
+  populate = ->(n) { create_list(:post, n) }
+  
+  assert_perform_linear_number_of_queries(slope: 1, populate: populate) do
+    Post.find_each { |p| p.user.name }
+  end
+end
+```
+
 You can also specify custom scale factors or filter patterns:
 
 ```ruby
@@ -133,6 +177,12 @@ assert_perform_constant_number_of_queries(
 end
 ```
 
+It's possible to specify a filter via `NPLUSONE_FILTER` env var, e.g.:
+
+```ruby
+NPLUSONE_FILTER = users bundle exec rake test
+```
+
 You can also specify `populate` as a test class instance method:
 
 ```ruby
@@ -145,6 +195,17 @@ def test_no_n_plus_one_error
     get :index
   end
 end
+
+```
+
+As in RSpec, you can use `current_scale` factor instead of `populate` block:
+
+```ruby
+def test_no_n_plus_one_error
+  assert_perform_constant_number_of_queries do
+    get :index, params: {per_page: current_scale}
+  end
+end
 ```
 
 ### With caching
@@ -156,9 +217,9 @@ If you use caching you can face the problem when first request performs more DB
 
 context "N + 1", :n_plus_one do
   populate { |n| create_list :post, n }
-  
+
   warmup { get :index } # cache something must be cached
-  
+
   specify do
     expect { get :index }.to perform_constant_number_of_queries
   end
@@ -185,7 +246,7 @@ end
 def test_no_n_plus_one
   populate = ->(n) { create_list(:post, n) }
   warmup = -> { get :index }
-  
+
   assert_perform_constant_number_of_queries population: populate, warmup: warmup do
     get :index
   end
@@ -193,6 +254,7 @@ end
 ```
 
 If your `warmup` and testing procs are identical, you can use:
+
 ```ruby
 expext { get :index }.to perform_constant_number_of_queries.with_warming_up # RSpec only
 ```
@@ -210,13 +272,21 @@ NPlusOneControl.default_scale_factors = [2, 3]
 # You can activate verbosity through env variable NPLUSONE_VERBOSE=1
 NPlusOneControl.verbose = false
 
+# Print table hits difference, for example:
+#
+#   Unmatched query numbers by tables:
+#     users (SELECT): 2 != 3
+#     events (INSERT): 1 != 2
+#
+self.show_table_stats = true
+
 # Ignore matching queries
 NPlusOneControl.ignore = /^(BEGIN|COMMIT|SAVEPOINT|RELEASE)/
 
 # ActiveSupport notifications event to track queries.
 # We track ActiveRecord event by default,
 # but can also track rom-rb events ('sql.rom') as well.
-NPlusOneControl.event = 'sql.active_record'
+NPlusOneControl.event = "sql.active_record"
 
 # configure transactional behavour for populate method
 # in case of use multiple database connections
@@ -230,6 +300,21 @@ NPlusOneControl::Executor.tap do |executor|
     connections.each(&:rollback_transaction)
   end
 end
+
+# Provide a backtrace cleaner callable object used to filter SQL caller location to display in the verbose mode
+# Set it to nil to disable tracing.
+#
+# In Rails apps, we use Rails.backtrace_cleaner by default.
+NPlusOneControl.backtrace_cleaner = ->(locations_array) { do_some_filtering(locations_array) }
+
+# You can also specify the number of backtrace lines to show.
+# MOTE: It could be specified via NPLUSONE_BACKTRACE env var
+NPlusOneControl.backtrace_length = 1
+
+# Sometime queries could be too large to provide any meaningful insight.
+# You can configure an output length limit for quries in verbose mode by setting the follwing option
+# NOTE: It could be specified via NPLUSONE_TRUNCATE env var
+NPlusOneControl.truncate_query_size = 100
 ```
 
 ## How does it work?
@@ -238,22 +323,29 @@ Take a look at our [Executor](https://github.com/palkan/n_plus_one_control/blob/
 
 ## What's next?
 
+- More matchers.
+
 It may be useful to provide more matchers/assertions, for example:
 
 ```ruby
 
 # Actually, that means that it is N+1))
-assert_linear_number_of_queries { ... }
+assert_linear_number_of_queries { some_code }
 
 # But we can tune it with `coef` and handle such cases as selecting in batches
 assert_linear_number_of_queries(coef: 0.1) do
-  Post.find_in_batches { ... }
+  Post.find_in_batches { some_code }
 end
 
 # probably, also make sense to add another curve types
-assert_logarithmic_number_of_queries { ... }
+assert_logarithmic_number_of_queries { some_code }
 ```
 
+- Support custom non-SQL events.
+
+N+1 problem is not a database specific: we can have N+1 Redis calls, N+1 HTTP external requests, etc.
+We can make `n_plus_one_control` customizable to support these scenarios (technically, we need to make it possible to handle different payload in the event subscriber).
+
 If you want to discuss or implement any of these, feel free to open an [issue](https://github.com/palkan/n_plus_one_control/issues) or propose a [pull request](https://github.com/palkan/n_plus_one_control/pulls).
 
 ## Development

data/lib/n_plus_one_control.rb

@@ -5,17 +5,99 @@ require "n_plus_one_control/executor"
 
 # RSpec and Minitest matchers to prevent N+1 queries problem.
 module NPlusOneControl
+  # Used to extract a table name from a query
+  EXTRACT_TABLE_RXP = /(insert into|update|delete from|from) ['"`](\S+)['"`]/i.freeze
+
+  # Used to convert a query part extracted by the regexp above to the corresponding
+  # human-friendly type
+  QUERY_PART_TO_TYPE = {
+    "insert into" => "INSERT",
+    "update" => "UPDATE",
+    "delete from" => "DELETE",
+    "from" => "SELECT"
+  }.freeze
+
   class << self
-    attr_accessor :default_scale_factors, :verbose, :ignore, :event
+    attr_accessor :default_scale_factors, :verbose, :show_table_stats, :ignore, :event,
+      :backtrace_cleaner, :backtrace_length, :truncate_query_size
+
+    attr_reader :default_matching
+
+    FAILURE_MESSAGES = {
+      constant_queries: "Expected to make the same number of queries",
+      linear_queries: "Expected to make linear number of queries"
+    }
 
-    def failure_message(queries)
-      msg = ["Expected to make the same number of queries, but got:\n"]
+    def failure_message(type, queries) # rubocop:disable Metrics/MethodLength
+      msg = ["#{FAILURE_MESSAGES[type]}, but got:\n"]
       queries.each do |(scale, data)|
         msg << "  #{data.size} for N=#{scale}\n"
-        msg << data.map { |sql| "    #{sql}\n" }.join.to_s if verbose
       end
+
+      msg.concat(table_usage_stats(queries.map(&:last))) if show_table_stats
+
+      if verbose
+        queries.each do |(scale, data)|
+          msg << "Queries for N=#{scale}\n"
+          msg << data.map { |sql| "  #{truncate_query(sql)}\n" }.join.to_s
+        end
+      end
+
       msg.join
     end
+
+    def table_usage_stats(runs) # rubocop:disable Metrics/MethodLength
+      msg = ["Unmatched query numbers by tables:\n"]
+
+      before, after = runs.map do |queries|
+        queries.group_by do |query|
+          matches = query.match(EXTRACT_TABLE_RXP)
+          next unless matches
+
+          "  #{matches[2]} (#{QUERY_PART_TO_TYPE[matches[1].downcase]})"
+        end.transform_values(&:count)
+      end
+
+      before.keys.each do |k|
+        next if before[k] == after[k]
+
+        msg << "#{k}: #{before[k]} != #{after[k]}\n"
+      end
+
+      msg
+    end
+
+    def default_matching=(val)
+      unless val
+        @default_matching = nil
+        return
+      end
+
+      @default_matching =
+        if val.is_a?(Regexp)
+          val
+        else
+          Regexp.new(val, Regexp::MULTILINE | Regexp::IGNORECASE)
+        end
+    end
+
+    private
+
+    def truncate_query(sql)
+      return sql unless truncate_query_size
+
+      # Only truncate query, leave tracing (if any) as is
+      parts = sql.split(/(\s+↳)/)
+
+      parts[0] =
+        if truncate_query_size < 4
+          "..."
+        else
+          parts[0][0..(truncate_query_size - 4)] + "..."
+        end
+
+      parts.join
+    end
   end
 
   # Scale factors to use.
@@ -23,7 +105,10 @@ module NPlusOneControl
   self.default_scale_factors = [2, 3]
 
   # Print performed queries if true
-  self.verbose = ENV['NPLUSONE_VERBOSE'] == '1'
+  self.verbose = ENV["NPLUSONE_VERBOSE"] == "1"
+
+  # Print table hits difference
+  self.show_table_stats = true
 
   # Ignore matching queries
   self.ignore = /^(BEGIN|COMMIT|SAVEPOINT|RELEASE)/
@@ -31,5 +116,16 @@ module NPlusOneControl
   # ActiveSupport notifications event to track queries.
   # We track ActiveRecord event by default,
   # but can also track rom-rb events ('sql.rom') as well.
-  self.event = 'sql.active_record'
+  self.event = "sql.active_record"
+
+  # Default query filtering applied if none provided explicitly
+  self.default_matching = ENV["NPLUSONE_FILTER"] || /^SELECT/i
+
+  # Truncate queries in verbose mode to fit the length
+  self.truncate_query_size = ENV["NPLUSONE_TRUNCATE"]&.to_i
+
+  # Define the number of backtrace lines to show
+  self.backtrace_length = ENV.fetch("NPLUSONE_BACKTRACE", 1).to_i
 end
+
+require "n_plus_one_control/railtie" if defined?(Rails::Railtie)

data/lib/n_plus_one_control/executor.rb

@@ -3,7 +3,7 @@
 module NPlusOneControl
   # Runs code for every scale factor
   # and returns collected queries.
-  module Executor
+  class Executor
     # Subscribes to ActiveSupport notifications and collect matching queries.
     class Collector
       def initialize(pattern)
@@ -19,46 +19,86 @@ module NPlusOneControl
         @queries
       end
 
-      def callback(_name, _start, _finish, _message_id, values)
+      def callback(_name, _start, _finish, _message_id, values) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/LineLength
         return if %w[CACHE SCHEMA].include? values[:name]
-        @queries << values[:sql] if @pattern.nil? || (values[:sql] =~ @pattern)
-      end
-    end
 
-    class << self
-      attr_accessor :transaction_begin
-      attr_accessor :transaction_rollback
+        return unless @pattern.nil? || (values[:sql] =~ @pattern)
 
-      def call(population:, scale_factors: nil, matching: nil)
-        raise ArgumentError, "Block is required!" unless block_given?
+        query = values[:sql]
 
-        results = []
-        collector = Collector.new(matching)
+        if NPlusOneControl.backtrace_cleaner && NPlusOneControl.verbose
+          source = extract_query_source_location(caller)
 
-        (scale_factors || NPlusOneControl.default_scale_factors).each do |scale|
-          with_transaction do
-            population.call(scale)
-            results << [scale, collector.call { yield }]
-          end
+          query = "#{query}\n    ↳ #{source.join("\n")}" unless source.empty?
         end
-        results
+
+        @queries << query
       end
 
       private
 
-      def with_transaction
-        transaction_begin.call
-        yield
-      ensure
-        transaction_rollback.call
+      def extract_query_source_location(locations)
+        NPlusOneControl.backtrace_cleaner.call(locations.lazy)
+          .take(NPlusOneControl.backtrace_length).to_a
       end
     end
 
+    class << self
+      attr_accessor :transaction_begin
+      attr_accessor :transaction_rollback
+    end
+
+    attr_reader :current_scale
+
     self.transaction_begin = -> do
       ActiveRecord::Base.connection.begin_transaction(joinable: false)
     end
+
     self.transaction_rollback = -> do
       ActiveRecord::Base.connection.rollback_transaction
     end
+
+    def initialize(population: nil, scale_factors: nil, matching: nil)
+      @population = population
+      @scale_factors = scale_factors
+      @matching = matching
+    end
+
+    # rubocop:disable Metrics/MethodLength
+    def call
+      raise ArgumentError, "Block is required!" unless block_given?
+
+      results = []
+      collector = Collector.new(matching)
+
+      (scale_factors || NPlusOneControl.default_scale_factors).each do |scale|
+        @current_scale = scale
+        with_transaction do
+          population&.call(scale)
+          results << [scale, collector.call { yield }]
+        end
+      end
+      results
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    private
+
+    def with_transaction
+      transaction_begin.call
+      yield
+    ensure
+      transaction_rollback.call
+    end
+
+    def transaction_begin
+      self.class.transaction_begin
+    end
+
+    def transaction_rollback
+      self.class.transaction_rollback
+    end
+
+    attr_reader :population, :scale_factors, :matching
   end
 end