n_plus_one_control 0.2.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 942639e351ee999dc4a113c237d9604878d296cc
-  data.tar.gz: 76d9b2b2fd28b02925ce4d219e3b5c6ac316d1fb
+SHA256:
+  metadata.gz: 394ec848692af46dd43c3a03a3315349c2aea19d174ff05ededa1ef825839ee0
+  data.tar.gz: 7e2a271fe8ae173b117d1fc514e33b6f08aa7c528e4ea476f5360591a28da6f8
 SHA512:
-  metadata.gz: 980d3feab9ebd9d9b51da847526a83ae0ed2b36e50509494d14bfad75578086b4c2ca6a6da60d5f518a6a2deb87c80af848c792046804e96b62249404f09535c
-  data.tar.gz: ef2d79c707712f2a578ba4ff513c1f03b6c4cf351b72381875188ac6a3730c1f73b3fd11ab7a0538f5d76f2d67a5b6fb4f57775218e1ac8b1a0877e6936bd0a9
+  metadata.gz: 1f411b0c1539f517e0c4036570e274fe62327817722065551fce09307e15402c31185566343ddb9fd388324a71c5c8d7730fc938767397f5bbba60f103ffdccb
+  data.tar.gz: 21a4445b6dd7efa9dad9bbfe2ccf8708ae813e2e60ea424b60f7b00cf58bef1971d58b1392893f34e817a3834792b461536890ed38619a20e0f9bd770ab7ddfd

data/CHANGELOG.md

@@ -0,0 +1,28 @@
+## master (unreleased)
+
+## 0.5.0 (2020-09-07)
+
+- **Ruby 2.5+ is required**. ([@palkan][])
+
+- Add support for multiple backtrace lines in verbose output. ([@palkan][])
+
+Could be specified via `NPLUSONE_BACKTRACE` env var.
+
+- Add `NPLUSONE_TRUNCATE` env var to truncate queries in verbose mode. ([@palkan][])
+
+- Support passing default filter via `NPLUSONE_FILTER` env var. ([@palkan][])
+
+- Add location tracing to SQLs in verbose mode. ([@palkan][])
+
+## 0.4.1 (2020-09-04)
+
+- Enhance failure message by showing differences in table hits. ([@palkan][])
+
+## 0.4.0 (2020-07-20)
+
+- Make scale factor available in tests via `#current_scale` method. ([@Earendil95][])
+
+- Start keeping a changelog. ([@palkan][])
+
+[@Earendil95]: https://github.com/Earendil95
+[@palkan]: https://github.com/palkan

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2017 palkan
+Copyright (c) 2017-2020 Vladimir Dementyev
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -1,4 +1,5 @@
-[![Gem Version](https://badge.fury.io/rb/n_plus_one_control.svg)](https://rubygems.org/gems/n_plus_one_control) [![Build Status](https://travis-ci.org/palkan/n_plus_one_control.svg?branch=master)](https://travis-ci.org/palkan/n_plus_one_control)
+[![Gem Version](https://badge.fury.io/rb/n_plus_one_control.svg)](https://rubygems.org/gems/n_plus_one_control)
+![Build](https://github.com/palkan/n_plus_one_control/workflows/Build/badge.svg)
 
 # N + 1 Control
 
@@ -31,7 +32,7 @@ Add this line to your application's Gemfile:
 
 ```ruby
 group :test do
-  gem 'n_plus_one_control'
+  gem "n_plus_one_control"
 end
 ```
 
@@ -47,8 +48,6 @@ First, add NPlusOneControl to your `spec_helper.rb`:
 
 ```ruby
 # spec_helper.rb
-...
-
 require "n_plus_one_control/rspec"
 ```
 
@@ -86,10 +85,30 @@ Availables modifiers:
 ```ruby
 # You can specify the RegExp to filter queries.
 # By default, it only considers SELECT queries.
-expect { ... }.to perform_constant_number_of_queries.matching(/INSERT/)
+expect { subject }.to perform_constant_number_of_queries.matching(/INSERT/)
 
 # You can also provide custom scale factors
-expect { ... }.to perform_constant_number_of_queries.with_scale_factors(10, 100)
+expect { subject }.to perform_constant_number_of_queries.with_scale_factors(10, 100)
+```
+
+#### Using scale factor in spec
+
+Let's suppose your action accepts parameter, which can make impact on the number of returned records:
+
+```ruby
+get :index, params: {per_page: 10}
+```
+
+Then it is enough to just change `per_page` parameter between executions and do not recreate records in DB. For this purpose, you can use `current_scale` method in your example:
+
+```ruby
+context "N+1", :n_plus_one do
+  before { create_list :post, 3 }
+
+  specify do
+    expect { get :index, params: {per_page: current_scale} }.to perform_constant_number_of_queries
+  end
+end
 ```
 
 ### Minitest
@@ -98,8 +117,6 @@ First, add NPlusOneControl to your `test_helper.rb`:
 
 ```ruby
 # test_helper.rb
-...
-
 require "n_plus_one_control/minitest"
 ```
 
@@ -133,6 +150,12 @@ assert_perform_constant_number_of_queries(
 end
 ```
 
+It's possible to specify a filter via `NPLUSONE_FILTER` env var, e.g.:
+
+```ruby
+NPLUSONE_FILTER = users bundle exec rake test
+```
+
 You can also specify `populate` as a test class instance method:
 
 ```ruby
@@ -145,6 +168,68 @@ def test_no_n_plus_one_error
     get :index
   end
 end
+
+```
+
+As in RSpec, you can use `current_scale` factor instead of `populate` block:
+
+```ruby
+def test_no_n_plus_one_error
+  assert_perform_constant_number_of_queries do
+    get :index, params: {per_page: current_scale}
+  end
+end
+```
+
+### With caching
+
+If you use caching you can face the problem when first request performs more DB queries than others. The solution is:
+
+```ruby
+# RSpec
+
+context "N + 1", :n_plus_one do
+  populate { |n| create_list :post, n }
+
+  warmup { get :index } # cache something must be cached
+
+  specify do
+    expect { get :index }.to perform_constant_number_of_queries
+  end
+end
+
+# Minitest
+
+def populate(n)
+  create_list(:post, n)
+end
+
+def warmup
+  get :index
+end
+
+def test_no_n_plus_one_error
+  assert_perform_constant_number_of_queries do
+    get :index
+  end
+end
+
+# or with params
+
+def test_no_n_plus_one
+  populate = ->(n) { create_list(:post, n) }
+  warmup = -> { get :index }
+
+  assert_perform_constant_number_of_queries population: populate, warmup: warmup do
+    get :index
+  end
+end
+```
+
+If your `warmup` and testing procs are identical, you can use:
+
+```ruby
+expext { get :index }.to perform_constant_number_of_queries.with_warming_up # RSpec only
 ```
 
 ### Configuration
@@ -160,13 +245,21 @@ NPlusOneControl.default_scale_factors = [2, 3]
 # You can activate verbosity through env variable NPLUSONE_VERBOSE=1
 NPlusOneControl.verbose = false
 
+# Print table hits difference, for example:
+#
+#   Unmatched query numbers by tables:
+#     users (SELECT): 2 != 3
+#     events (INSERT): 1 != 2
+#
+self.show_table_stats = true
+
 # Ignore matching queries
 NPlusOneControl.ignore = /^(BEGIN|COMMIT|SAVEPOINT|RELEASE)/
 
 # ActiveSupport notifications event to track queries.
 # We track ActiveRecord event by default,
 # but can also track rom-rb events ('sql.rom') as well.
-NPlusOneControl.event = 'sql.active_record'
+NPlusOneControl.event = "sql.active_record"
 
 # configure transactional behavour for populate method
 # in case of use multiple database connections
@@ -180,6 +273,21 @@ NPlusOneControl::Executor.tap do |executor|
     connections.each(&:rollback_transaction)
   end
 end
+
+# Provide a backtrace cleaner callable object used to filter SQL caller location to display in the verbose mode
+# Set it to nil to disable tracing.
+#
+# In Rails apps, we use Rails.backtrace_cleaner by default.
+NPlusOneControl.backtrace_cleaner = ->(locations_array) { do_some_filtering(locations_array) }
+
+# You can also specify the number of backtrace lines to show.
+# MOTE: It could be specified via NPLUSONE_BACKTRACE env var
+NPlusOneControl.backtrace_length = 1
+
+# Sometime queries could be too large to provide any meaningful insight.
+# You can configure an output length limit for quries in verbose mode by setting the follwing option
+# NOTE: It could be specified via NPLUSONE_TRUNCATE env var
+NPlusOneControl.truncate_query_size = 100
 ```
 
 ## How does it work?
@@ -188,22 +296,29 @@ Take a look at our [Executor](https://github.com/palkan/n_plus_one_control/blob/
 
 ## What's next?
 
+- More matchers.
+
 It may be useful to provide more matchers/assertions, for example:
 
 ```ruby
 
 # Actually, that means that it is N+1))
-assert_linear_number_of_queries { ... }
+assert_linear_number_of_queries { some_code }
 
 # But we can tune it with `coef` and handle such cases as selecting in batches
 assert_linear_number_of_queries(coef: 0.1) do
-  Post.find_in_batches { ... }
+  Post.find_in_batches { some_code }
 end
 
 # probably, also make sense to add another curve types
-assert_logarithmic_number_of_queries { ... }
+assert_logarithmic_number_of_queries { some_code }
 ```
 
+- Support custom non-SQL events.
+
+N+1 problem is not a database specific: we can have N+1 Redis calls, N+1 HTTP external requests, etc.
+We can make `n_plus_one_control` customizable to support these scenarios (technically, we need to make it possible to handle different payload in the event subscriber).
+
 If you want to discuss or implement any of these, feel free to open an [issue](https://github.com/palkan/n_plus_one_control/issues) or propose a [pull request](https://github.com/palkan/n_plus_one_control/pulls).
 
 ## Development

data/lib/n_plus_one_control.rb

@@ -5,17 +5,94 @@ require "n_plus_one_control/executor"
 
 # RSpec and Minitest matchers to prevent N+1 queries problem.
 module NPlusOneControl
+  # Used to extract a table name from a query
+  EXTRACT_TABLE_RXP = /(insert into|update|delete from|from) ['"](\S+)['"]/i.freeze
+
+  # Used to convert a query part extracted by the regexp above to the corresponding
+  # human-friendly type
+  QUERY_PART_TO_TYPE = {
+    "insert into" => "INSERT",
+    "update" => "UPDATE",
+    "delete from" => "DELETE",
+    "from" => "SELECT"
+  }.freeze
+
   class << self
-    attr_accessor :default_scale_factors, :verbose, :ignore, :event
+    attr_accessor :default_scale_factors, :verbose, :show_table_stats, :ignore, :event,
+      :backtrace_cleaner, :backtrace_length, :truncate_query_size
 
-    def failure_message(queries)
+    attr_reader :default_matching
+
+    def failure_message(queries) # rubocop:disable Metrics/MethodLength
       msg = ["Expected to make the same number of queries, but got:\n"]
       queries.each do |(scale, data)|
         msg << "  #{data.size} for N=#{scale}\n"
-        msg << data.map { |sql| "    #{sql}\n" }.join.to_s if verbose
       end
+
+      msg.concat(table_usage_stats(queries.map(&:last))) if show_table_stats
+
+      if verbose
+        queries.each do |(scale, data)|
+          msg << "Queries for N=#{scale}\n"
+          msg << data.map { |sql| "  #{truncate_query(sql)}\n" }.join.to_s
+        end
+      end
+
       msg.join
     end
+
+    def table_usage_stats(runs) # rubocop:disable Metrics/MethodLength
+      msg = ["Unmatched query numbers by tables:\n"]
+
+      before, after = runs.map do |queries|
+        queries.group_by do |query|
+          matches = query.match(EXTRACT_TABLE_RXP)
+          next unless matches
+
+          "  #{matches[2]} (#{QUERY_PART_TO_TYPE[matches[1].downcase]})"
+        end.transform_values(&:count)
+      end
+
+      before.keys.each do |k|
+        next if before[k] == after[k]
+
+        msg << "#{k}: #{before[k]} != #{after[k]}\n"
+      end
+
+      msg
+    end
+
+    def default_matching=(val)
+      unless val
+        @default_matching = nil
+        return
+      end
+
+      @default_matching =
+        if val.is_a?(Regexp)
+          val
+        else
+          Regexp.new(val, Regexp::MULTILINE | Regexp::IGNORECASE)
+        end
+    end
+
+    private
+
+    def truncate_query(sql)
+      return sql unless truncate_query_size
+
+      # Only truncate query, leave tracing (if any) as is
+      parts = sql.split(/(\s+↳)/)
+
+      parts[0] =
+        if truncate_query_size < 4
+          "..."
+        else
+          parts[0][0..(truncate_query_size - 4)] + "..."
+        end
+
+      parts.join
+    end
   end
 
   # Scale factors to use.
@@ -23,7 +100,10 @@ module NPlusOneControl
   self.default_scale_factors = [2, 3]
 
   # Print performed queries if true
-  self.verbose = ENV['NPLUSONE_VERBOSE'] == '1'
+  self.verbose = ENV["NPLUSONE_VERBOSE"] == "1"
+
+  # Print table hits difference
+  self.show_table_stats = true
 
   # Ignore matching queries
   self.ignore = /^(BEGIN|COMMIT|SAVEPOINT|RELEASE)/
@@ -31,5 +111,16 @@ module NPlusOneControl
   # ActiveSupport notifications event to track queries.
   # We track ActiveRecord event by default,
   # but can also track rom-rb events ('sql.rom') as well.
-  self.event = 'sql.active_record'
+  self.event = "sql.active_record"
+
+  # Default query filtering applied if none provided explicitly
+  self.default_matching = ENV["NPLUSONE_FILTER"] || /^SELECT/i
+
+  # Truncate queries in verbose mode to fit the length
+  self.truncate_query_size = ENV["NPLUSONE_TRUNCATE"]&.to_i
+
+  # Define the number of backtrace lines to show
+  self.backtrace_length = ENV.fetch("NPLUSONE_BACKTRACE", 1).to_i
 end
+
+require "n_plus_one_control/railtie" if defined?(Rails::Railtie)

data/lib/n_plus_one_control/executor.rb

@@ -3,7 +3,7 @@
 module NPlusOneControl
   # Runs code for every scale factor
   # and returns collected queries.
-  module Executor
+  class Executor
     # Subscribes to ActiveSupport notifications and collect matching queries.
     class Collector
       def initialize(pattern)
@@ -19,46 +19,86 @@ module NPlusOneControl
         @queries
       end
 
-      def callback(_name, _start, _finish, _message_id, values)
+      def callback(_name, _start, _finish, _message_id, values) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/LineLength
         return if %w[CACHE SCHEMA].include? values[:name]
-        @queries << values[:sql] if @pattern.nil? || (values[:sql] =~ @pattern)
-      end
-    end
 
-    class << self
-      attr_accessor :transaction_begin
-      attr_accessor :transaction_rollback
+        return unless @pattern.nil? || (values[:sql] =~ @pattern)
 
-      def call(population:, scale_factors: nil, matching: nil)
-        raise ArgumentError, "Block is required!" unless block_given?
+        query = values[:sql]
 
-        results = []
-        collector = Collector.new(matching)
+        if NPlusOneControl.backtrace_cleaner && NPlusOneControl.verbose
+          source = extract_query_source_location(caller)
 
-        (scale_factors || NPlusOneControl.default_scale_factors).each do |scale|
-          with_transaction do
-            population.call(scale)
-            results << [scale, collector.call { yield }]
-          end
+          query = "#{query}\n    ↳ #{source.join("\n")}" unless source.empty?
         end
-        results
+
+        @queries << query
       end
 
       private
 
-      def with_transaction
-        transaction_begin.call
-        yield
-      ensure
-        transaction_rollback.call
+      def extract_query_source_location(locations)
+        NPlusOneControl.backtrace_cleaner.call(locations.lazy)
+          .take(NPlusOneControl.backtrace_length).to_a
       end
     end
 
+    class << self
+      attr_accessor :transaction_begin
+      attr_accessor :transaction_rollback
+    end
+
+    attr_reader :current_scale
+
     self.transaction_begin = -> do
       ActiveRecord::Base.connection.begin_transaction(joinable: false)
     end
+
     self.transaction_rollback = -> do
       ActiveRecord::Base.connection.rollback_transaction
     end
+
+    def initialize(population: nil, scale_factors: nil, matching: nil)
+      @population = population
+      @scale_factors = scale_factors
+      @matching = matching
+    end
+
+    # rubocop:disable Metrics/MethodLength
+    def call
+      raise ArgumentError, "Block is required!" unless block_given?
+
+      results = []
+      collector = Collector.new(matching)
+
+      (scale_factors || NPlusOneControl.default_scale_factors).each do |scale|
+        @current_scale = scale
+        with_transaction do
+          population&.call(scale)
+          results << [scale, collector.call { yield }]
+        end
+      end
+      results
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    private
+
+    def with_transaction
+      transaction_begin.call
+      yield
+    ensure
+      transaction_rollback.call
+    end
+
+    def transaction_begin
+      self.class.transaction_begin
+    end
+
+    def transaction_rollback
+      self.class.transaction_rollback
+    end
+
+    attr_reader :population, :scale_factors, :matching
   end
 end