test_data 0.0.2 → 0.1.0

data/lib/test_data/statistics.rb

@@ -4,7 +4,7 @@ module TestData
   end
 
   class Statistics
-    attr_reader :load_count, :truncate_count
+    attr_reader :load_count, :truncate_count, :load_rails_fixtures_count
 
     def initialize
       reset
@@ -18,9 +18,14 @@ module TestData
       @truncate_count += 1
     end
 
+    def count_load_rails_fixtures!
+      @load_rails_fixtures_count += 1
+    end
+
     def reset
       @load_count = 0
       @truncate_count = 0
+      @load_rails_fixtures_count = 0
     end
   end
 end

data/lib/test_data/transactional_data_loader.rb

@@ -1,37 +1,132 @@
 module TestData
-  def self.load(transactions: true)
-    @transactional_data_loader ||= TransactionalDataLoader.new
-    @transactional_data_loader.load(transactions: transactions)
+  def self.load
+    @data_loader ||= TestData.config.use_transactional_data_loader ? TransactionalDataLoader.new : CommittingDataLoader.new
+    @data_loader.load
+  end
+
+  def self.ensure_we_dont_mix_transactional_and_non_transactional_data_loaders!(use_transactions)
+    unless @data_loader.nil? || use_transactions == @data_loader.transactional?
+      raise Error.new("There is already a #{@data_loader.transactional? ? "transactional" : "non-transactional"} data loader in use, and test_data does not support mixing both types of loaders in a single process.")
+    end
   end
 
   def self.rollback(save_point_name = :after_data_load)
-    @transactional_data_loader ||= TransactionalDataLoader.new
+    unless TestData.config.use_transactional_data_loader
+      raise Error.new("TestData.rollback requires config.use_transactional_data_loader = true")
+    end
+    @data_loader ||= TransactionalDataLoader.new
     case save_point_name
     when :before_data_load
-      @transactional_data_loader.rollback_to_before_data_load
+      @data_loader.rollback_to_before_data_load
     when :after_data_load
-      @transactional_data_loader.rollback_to_after_data_load
+      @data_loader.rollback_to_after_data_load
     when :after_data_truncate
-      @transactional_data_loader.rollback_to_after_data_truncate
+      @data_loader.rollback_to_after_data_truncate
+    when :after_load_rails_fixtures
+      @data_loader.rollback_to_after_load_rails_fixtures
     else
-      raise Error.new("No known save point named '#{save_point_name}'. Valid values are: [:before_data_load, :after_data_load, :after_data_truncate]")
+      raise Error.new("No known save point named '#{save_point_name}'. Valid values are: [:before_data_load, :after_data_load, :after_data_truncate, :after_load_rails_fixtures]")
     end
   end
 
-  def self.truncate(transactions: true)
-    @transactional_data_loader ||= TransactionalDataLoader.new
-    @transactional_data_loader.truncate(transactions: transactions)
+  def self.truncate
+    @data_loader ||= TestData.config.use_transactional_data_loader ? TransactionalDataLoader.new : CommittingDataLoader.new
+    @data_loader.truncate
+  end
+
+  def self.load_rails_fixtures(test_instance)
+    if !test_instance.respond_to?(:setup_fixtures)
+      raise Error.new("'TestData.load_rails_fixtures' must be passed a test instance that has had ActiveRecord::TestFixtures mixed-in (e.g. `TestData.load_rails_fixtures(self)` in an ActiveSupport::TestCase `setup` block), but the provided argument does not respond to 'setup_fixtures'")
+    elsif !test_instance.respond_to?(:__test_data_gem_setup_fixtures)
+      raise Error.new("'TestData.load_rails_fixtures' depends on Rails' default fixture-loading behavior being disabled by calling 'TestData.prevent_rails_fixtures_from_loading_automatically!' as early as possible (e.g. near the top of your test_helper.rb), but it looks like it was never called.")
+    elsif !TestData.config.use_transactional_data_loader
+      raise Error.new("'TestData.load_rails_fixtures' requires config.use_transactional_data_loader = true")
+    end
+    @data_loader ||= TransactionalDataLoader.new
+    @data_loader.load_rails_fixtures(test_instance)
   end
 
-  class TransactionalDataLoader
+  class AbstractDataLoader
     def initialize
       @config = TestData.config
       @statistics = TestData.statistics
+    end
+
+    protected
+
+    def execute_data_load
+      search_path = execute("show search_path").first["search_path"]
+      connection.disable_referential_integrity do
+        execute(File.read(@config.data_dump_full_path))
+      end
+      execute <<~SQL
+        select pg_catalog.set_config('search_path', '#{search_path}', false)
+      SQL
+      @statistics.count_load!
+    end
+
+    def execute_data_truncate
+      connection.disable_referential_integrity do
+        execute("TRUNCATE TABLE #{tables_to_truncate.map { |t| connection.quote_table_name(t) }.join(", ")} #{"CASCADE" unless @config.truncate_these_test_data_tables.present?}")
+      end
+      @statistics.count_truncate!
+    end
+
+    def execute_load_rails_fixtures(test_instance)
+      test_instance.pre_loaded_fixtures = false
+      test_instance.use_transactional_tests = false
+      test_instance.__test_data_gem_setup_fixtures
+      @already_loaded_rails_fixtures[test_instance.class] = test_instance.instance_variable_get(:@loaded_fixtures)
+      @statistics.count_load_rails_fixtures!
+    end
+
+    def tables_to_truncate
+      if @config.truncate_these_test_data_tables.present?
+        @config.truncate_these_test_data_tables
+      else
+        @tables_to_truncate ||= IO.foreach(@config.data_dump_path).grep(/^INSERT INTO/) { |line|
+          line.match(/^INSERT INTO ([^\s]+)/)&.captures&.first
+        }.compact.uniq
+      end
+    end
+
+    def connection
+      ActiveRecord::Base.connection
+    end
+
+    private
+
+    def execute(sql)
+      connection.execute(sql)
+    end
+  end
+
+  class CommittingDataLoader < AbstractDataLoader
+    def transactional?
+      false
+    end
+
+    def load
+      execute_data_load
+    end
+
+    def truncate
+      execute_data_truncate
+    end
+  end
+
+  class TransactionalDataLoader < AbstractDataLoader
+    def initialize
+      super
       @save_points = []
+      @already_loaded_rails_fixtures = {}
+    end
+
+    def transactional?
+      true
     end
 
-    def load(transactions: true)
-      return execute_data_load unless transactions
+    def load
       ensure_after_load_save_point_is_active_if_data_is_loaded!
       return rollback_to_after_data_load if save_point_active?(:after_data_load)
 
@@ -62,8 +157,14 @@ module TestData
       end
     end
 
-    def truncate(transactions: true)
-      return execute_data_truncate unless transactions
+    def rollback_to_after_load_rails_fixtures
+      if save_point_active?(:after_load_rails_fixtures)
+        rollback_save_point(:after_load_rails_fixtures)
+        create_save_point(:after_load_rails_fixtures)
+      end
+    end
+
+    def truncate
       ensure_after_load_save_point_is_active_if_data_is_loaded!
       ensure_after_truncate_save_point_is_active_if_data_is_truncated!
       return rollback_to_after_data_truncate if save_point_active?(:after_data_truncate)
@@ -81,8 +182,8 @@ module TestData
         # should expect that the existence of :after_data_truncate save point
         # implies that it's safe to rollback to the :after_data_load save
         # point; since tests run in random order, it's likely to happen
-        TestData.log.debug("TestData.truncate was called, but data was not loaded. Loading data before truncate to preserve the documents transaction save point ordering")
-        load(transactions: true)
+        TestData.log.debug("TestData.truncate was called, but data was not loaded. Loading data before truncate to preserve the transaction save point ordering")
+        load
       end
 
       execute_data_truncate
@@ -90,6 +191,26 @@ module TestData
       create_save_point(:after_data_truncate)
     end
 
+    # logic is beat-for-beat the same as #truncate just one step deeper down
+    # this rabbit hole…
+    def load_rails_fixtures(test_instance)
+      ensure_after_load_save_point_is_active_if_data_is_loaded!
+      ensure_after_truncate_save_point_is_active_if_data_is_truncated!
+      ensure_after_load_rails_fixtures_save_point_is_active_if_fixtures_are_loaded!
+      reset_rails_fixture_caches(test_instance)
+      return rollback_to_after_load_rails_fixtures if save_point_active?(:after_load_rails_fixtures) && @already_loaded_rails_fixtures[test_instance.class].present?
+
+      if save_point_active?(:after_data_truncate)
+        rollback_to_after_data_truncate
+      else
+        truncate
+      end
+
+      execute_load_rails_fixtures(test_instance)
+      record_ar_internal_metadata_that_rails_fixtures_are_loaded
+      create_save_point(:after_load_rails_fixtures)
+    end
+
     private
 
     def ensure_after_load_save_point_is_active_if_data_is_loaded!
@@ -130,32 +251,29 @@ module TestData
       ActiveRecord::InternalMetadata.find_by(key: "test_data:truncated")&.value == "true"
     end
 
-    def execute_data_load
-      search_path = execute("show search_path").first["search_path"]
-      connection.disable_referential_integrity do
-        execute(File.read(@config.data_dump_full_path))
+    def ensure_after_load_rails_fixtures_save_point_is_active_if_fixtures_are_loaded!
+      if !save_point_active?(:after_load_rails_fixtures) && ar_internal_metadata_shows_rails_fixtures_are_loaded?
+        TestData.log.debug "Rails Fixtures appears to have been loaded by test_data, but the :after_load_rails_fixtures save point was rolled back (and not by this gem). Recreating the :after_load_rails_fixtures save point"
+        create_save_point(:after_load_rails_fixtures)
       end
-      execute <<~SQL
-        select pg_catalog.set_config('search_path', '#{search_path}', false)
-      SQL
-      @statistics.count_load!
     end
 
-    def execute_data_truncate
-      connection.disable_referential_integrity do
-        execute("TRUNCATE TABLE #{tables_to_truncate.map { |t| connection.quote_table_name(t) }.join(", ")}")
+    def record_ar_internal_metadata_that_rails_fixtures_are_loaded
+      if ar_internal_metadata_shows_rails_fixtures_are_loaded?
+        TestData.log.warn "Attempted to record that test_data had loaded your Rails fixtures in ar_internal_metadata, but record already existed. Perhaps a previous test run committed the loading of your Rails fixtures?"
+      else
+        ActiveRecord::InternalMetadata.create!(key: "test_data:rails_fixtures_loaded", value: "true")
       end
-      @statistics.count_truncate!
     end
 
-    def tables_to_truncate
-      if @config.truncate_these_test_data_tables.present?
-        @config.truncate_these_test_data_tables
-      else
-        @tables_to_truncate ||= IO.foreach(@config.data_dump_path).grep(/^INSERT INTO/) { |line|
-          line.match(/^INSERT INTO ([^\s]+)/)&.captures&.first
-        }.compact.uniq
-      end
+    def ar_internal_metadata_shows_rails_fixtures_are_loaded?
+      ActiveRecord::InternalMetadata.find_by(key: "test_data:rails_fixtures_loaded")&.value == "true"
+    end
+
+    def reset_rails_fixture_caches(test_instance)
+      ActiveRecord::FixtureSet.reset_cache
+      test_instance.instance_variable_set(:@loaded_fixtures, @already_loaded_rails_fixtures[test_instance.class])
+      test_instance.instance_variable_set(:@fixture_cache, {})
     end
 
     def save_point_active?(name)
@@ -178,13 +296,5 @@ module TestData
         save_point.active?
       }
     end
-
-    def execute(sql)
-      connection.execute(sql)
-    end
-
-    def connection
-      ActiveRecord::Base.connection
-    end
   end
 end

data/lib/test_data/version.rb

@@ -1,3 +1,3 @@
 module TestData
-  VERSION = "0.0.2"
+  VERSION = "0.1.0"
 end

data/script/test

@@ -35,6 +35,9 @@ bin/rails test test/integration/updated_boops_test.rb
 
 # Test a migration being added and run and an out-of-date dump being loaded
 cp ../test/fixtures/20210418220133_add_beep_to_boops.rb db/migrate
+cp ../test/fixtures/20210624180810_create_pants.rb db/migrate
+cp ../test/fixtures/pant.rb app/models
+cp ../test/fixtures/pants.yml test/fixtures
 bin/rake db:migrate
 bin/rake db:test:prepare
 bin/rake test_data:drop_database
@@ -46,6 +49,12 @@ bin/rails test test/integration/migrated_boops_test.rb
 # Run a test that commits test data thru to the database
 bin/rails test test/integration/transaction_committing_boops_test.rb
 
+# Run a test that prevents Rails fixtures for preloading and then loads them in a transaction
+bin/rails test test/integration/rails_fixtures_override_test.rb
+
+# Run a test that forgets to prevent Rails fixtures but then tries to load them in a transaction
+bin/rails test test/integration/rails_fixtures_double_load_test.rb
+
 # Add a second migration, this time without wiping the test_data db and with a table we want to ignore
 cp ../test/fixtures/20210423114916_add_table_we_want_to_ignore.rb db/migrate
 cp ../test/fixtures/chatty_audit_log.rb app/models
@@ -88,3 +97,5 @@ bin/rails test test/integration/boops_that_boop_boops_test.rb
 # Cleanup
 cd ..
 ./script/reset_example_app
+
+echo "You win!"

data/test_data.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
 
   spec.metadata["homepage_uri"] = spec.homepage
   spec.metadata["source_code_uri"] = spec.homepage
-  spec.metadata["changelog_uri"] = "#{spec.homepage}/blob/master/CHANGELOG.md"
+  spec.metadata["changelog_uri"] = "#{spec.homepage}/blob/main/CHANGELOG.md"
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: test_data
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
 platform: ruby
 authors:
 - Justin Searls
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-04-30 00:00:00.000000000 Z
+date: 2021-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: railties
@@ -106,25 +106,33 @@ files:
 - example/test/integration/mode_switching_demo_test.rb
 - example/test/integration/parallel_boops_with_fixtures_test.rb
 - example/test/integration/parallel_boops_without_fixtures_test.rb
+- example/test/integration/rails_fixtures_double_load_test.rb
+- example/test/integration/rails_fixtures_override_test.rb
 - example/test/integration/transaction_committing_boops_test.rb
 - example/test/integration/updated_boops_test.rb
 - example/test/test_helper.rb
+- lib/generators/test_data/cable_yaml_generator.rb
 - lib/generators/test_data/database_yaml_generator.rb
 - lib/generators/test_data/environment_file_generator.rb
 - lib/generators/test_data/initializer_generator.rb
+- lib/generators/test_data/secrets_yaml_generator.rb
 - lib/generators/test_data/webpacker_yaml_generator.rb
 - lib/test_data.rb
+- lib/test_data/active_record_ext.rb
 - lib/test_data/active_support_ext.rb
 - lib/test_data/config.rb
 - lib/test_data/configuration_verification.rb
 - lib/test_data/configurators.rb
+- lib/test_data/configurators/cable_yaml.rb
 - lib/test_data/configurators/database_yaml.rb
 - lib/test_data/configurators/environment_file.rb
 - lib/test_data/configurators/initializer.rb
+- lib/test_data/configurators/secrets_yaml.rb
 - lib/test_data/configurators/webpacker_yaml.rb
 - lib/test_data/detects_database_emptiness.rb
 - lib/test_data/dumps_database.rb
 - lib/test_data/error.rb
+- lib/test_data/generator_support.rb
 - lib/test_data/installs_configuration.rb
 - lib/test_data/loads_database_dumps.rb
 - lib/test_data/log.rb
@@ -144,7 +152,7 @@ licenses: []
 metadata:
   homepage_uri: https://github.com/testdouble/test_data
   source_code_uri: https://github.com/testdouble/test_data
-  changelog_uri: https://github.com/testdouble/test_data/blob/master/CHANGELOG.md
+  changelog_uri: https://github.com/testdouble/test_data/blob/main/CHANGELOG.md
 post_install_message:
 rdoc_options: []
 require_paths: