test-prof 0.11.0 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e2060dd26a4bec62bd9a3b699e1165c60b8914a74c9fc25b3a0edf84afbd7d2b
-  data.tar.gz: 9b8e3eb78764c130ae78a575a5cdfe5d9e9a440d7b748c108b5088d99dc7261d
+  metadata.gz: 64b1be0cdb0b04518ed2fb900756fd07d212f79709d76e784798384fb92a2a5f
+  data.tar.gz: fa59a7d025eda4e53ce0381cc9c973cbcad50e0c4584b1ca30aa5cfb1e9d0ebc
 SHA512:
-  metadata.gz: 6998b3cb30fa82bf6b2311dfa733f0c32694964741aa923b1300115d670c0cfc7e834ad55a9a9b4295e5b02e85d18e529d75d775b16d00ecc44956bee30231aa
-  data.tar.gz: b42a91195b9e6a49522bd33ede09ae9ee769b7ee1678de97d2a0bcf593ab076811f20938f2f2deabe864e22e6c9cab5b0e1b7431da91492b9d7959e7b6b52769
+  metadata.gz: d08d874b6948df045adbdba66960abbb47e2fe8fd55700b8f9f4ab6a411f94ffd632d15937c30df36728709440be7209f21951a47b66f54c1ea39669aed35b72
+  data.tar.gz: 955199e5fce56fea5237bc07151526e83156ebd7f3f74f00a05eac5b7da20ad9b7ffe21807c8c7644327bc13020315d8262436e65f323057fd75be4378a89342

data/CHANGELOG.md

@@ -1,6 +1,55 @@
 # Change log
 
-## master (unreleased)
+## 0.12.0 (2020-07-17)
+
+- Add state leakage detection for `let_it_be`. ([@pirj][], [@jaimerson][], [@alexvko][])
+
+- Add default let_it_be modifiers configuration. ([@palkan][])
+
+  You can configure global modifiers:
+
+  ```ruby
+  TestProf::LetItBe.configure do |config|
+    # Make refind activated by default
+    config.default_modifiers[:refind] = true
+  end
+  ```
+
+  Or for specific contexts via tags:
+
+  ```ruby
+  context "with let_it_be reload", let_it_be_modifiers: {reload: true} do
+    # examples
+  end
+  ```
+
+- **Drop Ruby 2.4 support.** ([@palkan][])
+
+- SAMPLE and SAMPLE_GROUP work consistently with seed in RSpec and Minitest. ([@stefkin][])
+
+- Make sure EventProf is not affected by time freezing. ([@palkan][])
+
+  EventProf results now is not affected by `Timecop.freeze` or similar.
+
+  See more in [#181](https://github.com/palkan/test-prof/issues/181).
+
+- Adds the ability to define stackprof's interval sampling by using `TEST_STACK_PROF_INTERVAL` env variable ([@LynxEyes][])
+
+  Now you can use `$ TEST_STACK_PROF=1 TEST_STACK_PROF_INTERVAL=10000 rspec` to define a custom interval (in microseconds).
+
+## 0.11.3 (2020-02-11)
+
+- Disable `RSpec/AggregateFailures` by default. ([@pirj][])
+
+## 0.11.2 (2020-02-11)
+
+- Fix RuboCop integration regressions. ([@palkan][])
+
+## 0.11.1 (2020-02-10)
+
+- Add `config/` to the gem contents. ([@palkan][])
+
+  Fixes RuboCop integration regression from 0.11.0.
 
 ## 0.11.0 (2020-02-09)
 
@@ -522,3 +571,7 @@ Fixes [#10](https://github.com/palkan/test-prof/issues/10).
 [@tyleriguchi]: https://github.com/tyleriguchi
 [@lostie]: https://github.com/lostie
 [@pirj]: https://github.com/pirj
+[@LynxEyes]: https://github.com/LynxEyes
+[@stefkin]: https://github.com/stefkin
+[@jaimerson]: https://github.com/jaimerson
+[@alexvko]: https://github.com/alexvko

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2017-2019 palkan
+Copyright (c) 2017-2020 Vladimir Dementyev
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -1,4 +1,4 @@
-[![Cult Of Martians](http://cultofmartians.com/assets/badges/badge.svg)](http://cultofmartians.com)
+[![Cult Of Martians](http://cultofmartians.com/assets/badges/badge.svg)](https://cultofmartians.com)
 [![Gem Version](https://badge.fury.io/rb/test-prof.svg)](https://rubygems.org/gems/test-prof) [![Build](https://github.com/palkan/test-prof/workflows/Build/badge.svg)](https://github.com/palkan/test-prof/actions)
 [![JRuby Build](https://github.com/palkan/test-prof/workflows/JRuby%20Build/badge.svg)](https://github.com/palkan/test-prof/actions)
 [![Code Triagers Badge](https://www.codetriage.com/palkan/test-prof/badges/users.svg)](https://www.codetriage.com/palkan/test-prof)
@@ -47,7 +47,7 @@ TestProf toolbox aims to help you identify bottlenecks in your test suite. It co
 ## Who uses TestProf
 
 - [Discourse](https://github.com/discourse/discourse) reduced [~27% of their test suite time](https://twitter.com/samsaffron/status/1125602558024699904)
-- [Gitlab](https://gitlab.com/gitlab-org/gitlab-ce) reduced [39% of their API tests time](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14370)
+- [Gitlab](https://gitlab.com/gitlab-org/gitlab-ce) reduced [39% of their API tests time](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14370) and [improved factories usage](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26810)
 - [CodeTriage](https://github.com/codetriage/codetriage)
 - [Dev.to](https://github.com/thepracticaldev/dev.to)
 - [Open Project](https://github.com/opf/openproject)
@@ -104,7 +104,3 @@ Already using TestProf? [Share your story!](https://github.com/palkan/test-prof/
 The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
 
 [docs]: https://test-prof.evilmartians.io
-
-## Security Contact
-
-To report a security vulnerability, please use the [Tidelift security contact](https://tidelift.com/security). Tidelift will coordinate the fix and disclosure.

data/config/default.yml

@@ -0,0 +1,35 @@
+---
+AllCops:
+  RSpec:
+    Patterns:
+    - _spec.rb
+    - "(?:^|/)spec/"
+
+RSpec/AggregateExamples:
+  Description: Checks if example group contains two or more aggregatable examples.
+  Enabled: true
+  StyleGuide: https://rspec.rubystyle.guide/#expectation-per-example
+  AddAggregateFailuresMetadata: true
+  MatchersWithSideEffects:
+    - allow_value
+    - allow_values
+    - validate_presence_of
+    - validate_absence_of
+    - validate_length_of
+    - validate_inclusion_of
+    - validates_exclusion_of
+
+# TODO: remove this one we hit 1.0
+RSpec/AggregateFailures:
+  Description: Checks if example group contains two or more aggregatable examples.
+  Enabled: false
+  StyleGuide: https://rspec.rubystyle.guide/#expectation-per-example
+  AddAggregateFailuresMetadata: true
+  MatchersWithSideEffects:
+    - allow_value
+    - allow_values
+    - validate_presence_of
+    - validate_absence_of
+    - validate_length_of
+    - validate_inclusion_of
+    - validates_exclusion_of

data/lib/minitest/test_prof_plugin.rb

@@ -12,6 +12,7 @@ module Minitest # :nodoc:
         opts[:top_count] = ENV["EVENT_PROF_TOP"].to_i if ENV["EVENT_PROF_TOP"]
         opts[:per_example] = true if ENV["EVENT_PROF_EXAMPLES"]
         opts[:fdoc] = true if ENV["FDOC"]
+        opts[:sample] = true if ENV["SAMPLE"] || ENV["SAMPLE_GROUPS"]
       end
     end
   end
@@ -39,5 +40,7 @@ module Minitest # :nodoc:
 
     reporter << TestProf::EventProfReporter.new(options[:io], options) if options[:event]
     reporter << TestProf::FactoryDoctorReporter.new(options[:io], options) if options[:fdoc]
+
+    ::TestProf::MinitestSample.call if options[:sample]
   end
 end

data/lib/test_prof/before_all.rb

@@ -24,10 +24,6 @@ module TestProf
         yield
       end
 
-      def within_transaction
-        yield
-      end
-
       def rollback_transaction
         raise AdapterMissing if adapter.nil?
 

data/lib/test_prof/cops/inject.rb

@@ -2,22 +2,24 @@
 
 # This is shamelessly borrowed from RuboCop RSpec
 # https://github.com/rubocop-hq/rubocop-rspec/blob/master/lib/rubocop/rspec/inject.rb
-module RuboCop
-  # Because RuboCop doesn't yet support plugins, we have to monkey patch in a
-  # bit of our configuration.
-  module Inject
-    PROJECT_ROOT = Pathname.new(__dir__).parent.parent.parent.expand_path.freeze
-    CONFIG_DEFAULT = PROJECT_ROOT.join("config", "default.yml").freeze
+module TestProf
+  module Cops
+    # Because RuboCop doesn't yet support plugins, we have to monkey patch in a
+    # bit of our configuration.
+    module Inject
+      PROJECT_ROOT = Pathname.new(__dir__).parent.parent.parent.expand_path.freeze
+      CONFIG_DEFAULT = PROJECT_ROOT.join("config", "default.yml").freeze
 
-    def self.defaults!
-      path = CONFIG_DEFAULT.to_s
-      hash = ConfigLoader.send(:load_yaml_configuration, path)
-      config = Config.new(hash, path)
-      puts "configuration from #{path}" if ConfigLoader.debug?
-      config = ConfigLoader.merge_with_default(config, path)
-      ConfigLoader.instance_variable_set(:@default_configuration, config)
+      def self.defaults!
+        path = CONFIG_DEFAULT.to_s
+        hash = RuboCop::ConfigLoader.send(:load_yaml_configuration, path)
+        config = RuboCop::Config.new(hash, path)
+        puts "configuration from #{path}" if RuboCop::ConfigLoader.debug?
+        config = RuboCop::ConfigLoader.merge_with_default(config, path)
+        RuboCop::ConfigLoader.instance_variable_set(:@default_configuration, config)
+      end
     end
   end
 end
 
-RuboCop::Inject.defaults!
+TestProf::Cops::Inject.defaults!

data/lib/test_prof/cops/rspec/aggregate_examples.rb

@@ -12,7 +12,7 @@ module RuboCop
     module RSpec
       # Checks if example groups contain two or more aggregatable examples.
       #
-      # @see https://github.com/rubocop-hq/rspec-style-guide#expectations-per-example
+      # @see https://github.com/rubocop-hq/rspec-style-guide#expectation-per-example
       #
       # This cop is primarily for reducing the cost of repeated expensive
       # context initialization.

data/lib/test_prof/cops/rspec/aggregate_failures.rb

@@ -4,14 +4,12 @@ module RuboCop
   module Cop
     module RSpec
       class AggregateExamples
-        def self.inherited(subclass)
-          superclass.registry.enlist(subclass)
+        def self.registry
+          RuboCop::Cop::Cop.registry
         end
       end
 
       class AggregateFailures < AggregateExamples
-        raise "Remove me" if TestProf::VERSION >= "1.0"
-
         def initialize(*)
           super
           self.class.just_once { warn "`AggregateFailures` cop has been renamed to `AggregateExamples`." }

data/lib/test_prof/event_prof/instrumentations/active_support.rb

@@ -4,13 +4,31 @@ module TestProf::EventProf
   module Instrumentations
     # Wrapper over ActiveSupport::Notifications
     module ActiveSupport
+      class Subscriber
+        attr_reader :block, :started_at
+
+        def initialize(block)
+          @block = block
+        end
+
+        def start(*)
+          @started_at = TestProf.now
+        end
+
+        def publish(_name, started_at, finished_at, *)
+          block.call(finished_at - started_at)
+        end
+
+        def finish(*)
+          block.call(TestProf.now - started_at)
+        end
+      end
+
       class << self
-        def subscribe(event)
+        def subscribe(event, &block)
           raise ArgumentError, "Block is required!" unless block_given?
 
-          ::ActiveSupport::Notifications.subscribe(event) do |_event, start, finish, *_args|
-            yield (finish - start)
-          end
+          ::ActiveSupport::Notifications.subscribe(event, Subscriber.new(block))
         end
 
         def instrument(event)

data/lib/test_prof/recipes/minitest/sample.rb

@@ -35,18 +35,14 @@ module TestProf
           end
         end
       end
-    end
 
-    # Overrides Minitest.run
-    def run(*)
-      if ENV["SAMPLE"]
-        MinitestSample.sample_examples(ENV["SAMPLE"].to_i)
-      elsif ENV["SAMPLE_GROUPS"]
-        MinitestSample.sample_groups(ENV["SAMPLE_GROUPS"].to_i)
+      def call
+        if ENV["SAMPLE"]
+          ::TestProf::MinitestSample.sample_examples(ENV["SAMPLE"].to_i)
+        elsif ENV["SAMPLE_GROUPS"]
+          ::TestProf::MinitestSample.sample_groups(ENV["SAMPLE_GROUPS"].to_i)
+        end
       end
-      super
     end
   end
 end
-
-Minitest.singleton_class.prepend(TestProf::MinitestSample)

data/lib/test_prof/recipes/rspec/before_all.rb

@@ -9,7 +9,7 @@ module TestProf
       def before_all(&block)
         raise ArgumentError, "Block is required!" unless block_given?
 
-        return within_before_all(&block) if within_before_all?
+        return before(:all, &block) if within_before_all?
 
         @__before_all_activated__ = true
 
@@ -24,14 +24,6 @@ module TestProf
         end
       end
 
-      def within_before_all(&block)
-        before(:all) do
-          BeforeAll.within_transaction do
-            instance_eval(&block)
-          end
-        end
-      end
-
       def within_before_all?
         instance_variable_defined?(:@__before_all_activated__)
       end

data/lib/test_prof/recipes/rspec/let_it_be.rb

@@ -22,6 +22,10 @@ module TestProf
 
         LetItBe.modifiers[key] = block
       end
+
+      def default_modifiers
+        @default_modifiers ||= {}
+      end
     end
 
     class << self
@@ -75,6 +79,8 @@ module TestProf
     # And we love cats!)
     PREFIX = RUBY_ENGINE == "jruby" ? "@__jruby_is_not_cat_friendly__" : "@😸"
 
+    FROZEN_ERROR_HINT = "\nIf you are using `let_it_be`, you may want to pass `reload: true` or `refind: true` modifier to it."
+
     def self.define_let_it_be_alias(name, **default_args)
       define_method(name) do |identifier, **options, &blk|
         let_it_be(identifier, **default_args.merge(options), &blk)
@@ -83,20 +89,20 @@ module TestProf
 
     def let_it_be(identifier, **options, &block)
       initializer = proc do
-        instance_variable_set(:"#{PREFIX}#{identifier}", instance_exec(&block))
+        instance_variable_set(:"#{TestProf::LetItBe::PREFIX}#{identifier}", instance_exec(&block))
+      rescue FrozenError => e
+        e.message << TestProf::LetItBe::FROZEN_ERROR_HINT
+        raise
       end
 
-      if within_before_all?
-        within_before_all(&initializer)
-      else
-        before_all(&initializer)
-      end
+      default_options = LetItBe.config.default_modifiers.dup
+      default_options.merge!(metadata[:let_it_be_modifiers]) if metadata[:let_it_be_modifiers]
 
-      define_let_it_be_methods(identifier, **options)
-    end
+      options = default_options.merge(options)
+
+      before_all(&initializer)
 
-    def define_let_it_be_methods(identifier, **modifiers)
-      let_accessor = LetItBe.wrap_with_modifiers(modifiers) do
+      let_accessor = LetItBe.wrap_with_modifiers(options) do
         instance_variable_get(:"#{PREFIX}#{identifier}")
       end
 
@@ -114,16 +120,78 @@ module TestProf
 
       let(identifier, &let_accessor)
     end
+
+    module Freezer
+      # Stoplist to prevent freezing objects and theirs associations that are defined
+      # with `let_it_be`'s `freeze: false` options during deep freezing.
+      #
+      # To only keep track of objects that are available in current example group,
+      # `begin` adds a new layer, and `rollback` removes a layer of unrelated objects
+      # along with rolling back the transaction where they were created.
+      #
+      # Stoplist holds records declared with `freeze: false` (so we do not freeze them even if they're used as
+      # associated records for frozen objects)
+      module Stoplist
+        class << self
+          def stop?(record)
+            @stoplist.any? { |layer| layer.include?(record) }
+          end
+
+          def stop!(record)
+            @stoplist.last.push(record)
+          end
+
+          def begin
+            @stoplist.push([])
+          end
+
+          def rollback
+            @stoplist.pop
+          end
+        end
+
+        # Stack of example group-related variable definitions
+        @stoplist = []
+      end
+
+      class << self
+        # Rerucsively freezes the object to detect modifications
+        def deep_freeze(record)
+          return if record.frozen?
+          return if Stoplist.stop?(record)
+
+          record.freeze
+
+          # Support `let_it_be` with `create_list`
+          return record.each { |rec| deep_freeze(rec) } if record.respond_to?(:each)
+
+          # Freeze associations as well.
+          return unless defined?(::ActiveRecord::Base)
+          return unless record.is_a?(::ActiveRecord::Base)
+
+          record.class.reflections.keys.each do |reflection|
+            # But only if they are already loaded. If not yet loaded, they weren't
+            # created by factories, and it's ok to mutate them.
+
+            next unless record.association(reflection.to_sym).loaded?
+
+            target = record.association(reflection.to_sym).target
+            deep_freeze(target) if target.is_a?(::ActiveRecord::Base) || target.respond_to?(:each)
+          end
+        end
+      end
+    end
   end
 end
 
-if defined?(::ActiveRecord)
+if defined?(::ActiveRecord::Base)
   require "test_prof/ext/active_record_refind"
   using TestProf::Ext::ActiveRecordRefind
 
   TestProf::LetItBe.configure do |config|
     config.register_modifier :reload do |record, val|
       next record unless val
+
       next record.reload if record.is_a?(::ActiveRecord::Base)
 
       if record.respond_to?(:map)
@@ -136,6 +204,7 @@ if defined?(::ActiveRecord)
 
     config.register_modifier :refind do |record, val|
       next record unless val
+
       next record.refind if record.is_a?(::ActiveRecord::Base)
 
       if record.respond_to?(:map)
@@ -145,7 +214,35 @@ if defined?(::ActiveRecord)
       end
       record
     end
+
+    config.register_modifier :freeze do |record, val|
+      if val == false
+        TestProf::LetItBe::Freezer::Stoplist.stop!(record)
+        next record
+      end
+
+      TestProf::LetItBe::Freezer.deep_freeze(record)
+      record
+    end
   end
 end
 
 RSpec::Core::ExampleGroup.extend TestProf::LetItBe
+
+TestProf::BeforeAll.configure do |config|
+  config.before(:begin) do
+    TestProf::LetItBe::Freezer::Stoplist.begin
+  end
+
+  config.after(:rollback) do
+    TestProf::LetItBe::Freezer::Stoplist.rollback
+  end
+end
+
+RSpec.configure do |config|
+  config.after(:example) do |example|
+    if example.exception&.is_a?(FrozenError)
+      example.exception.message << TestProf::LetItBe::FROZEN_ERROR_HINT
+    end
+  end
+end

data/lib/test_prof/recipes/rspec/sample.rb

@@ -15,7 +15,8 @@ if ENV["SAMPLE"]
   RSpec.configure do |config|
     config.before(:suite) do
       filtered_examples = RSpec.world.filtered_examples.values.flatten
-      sample = filtered_examples.sample(ENV["SAMPLE"].to_i)
+      random = Random.new(RSpec.configuration.seed)
+      sample = filtered_examples.sample(ENV["SAMPLE"].to_i, random: random)
       RSpec.world.filtered_examples = Hash.new do |hash, group|
         hash[group] = group.examples & sample
       end
@@ -31,7 +32,8 @@ if ENV["SAMPLE_GROUPS"]
       filtered_groups = RSpec.world.filtered_examples.reject do |_group, examples|
         examples.empty?
       end.keys
-      sample = filtered_groups.sample(ENV["SAMPLE_GROUPS"].to_i)
+      random = Random.new(RSpec.configuration.seed)
+      sample = filtered_groups.sample(ENV["SAMPLE_GROUPS"].to_i, random: random)
       RSpec.world.filtered_examples = Hash.new do |hash, group|
         hash[group] = sample.include?(group) ? group.examples : []
       end

data/lib/test_prof/stack_prof.rb

@@ -36,6 +36,9 @@ module TestProf
           else
             "html"
           end
+
+        sample_interval = ENV["TEST_STACK_PROF_INTERVAL"].to_i
+        @interval = sample_interval > 0 ? sample_interval : nil
       end
 
       def raw?

data/lib/test_prof/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module TestProf
-  VERSION = "0.11.0"
+  VERSION = "0.12.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: test-prof
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.12.0
 platform: ruby
 authors:
 - Vladimir Dementyev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-09 00:00:00.000000000 Z
+date: 2020-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -56,42 +56,42 @@ dependencies:
   name: isolator
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0.6'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '5.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '5.9'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.77.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.77.0
 description: "\n    Ruby applications tests profiling tools.\n\n    Contains tools
@@ -118,6 +118,7 @@ files:
 - assets/src/d3.v4.min.js
 - assets/tagprof.demo.html
 - assets/tagprof.template.html
+- config/default.yml
 - lib/minitest/base_reporter.rb
 - lib/minitest/event_prof_formatter.rb
 - lib/minitest/test_prof_plugin.rb
@@ -224,7 +225,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.4.0
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="