test_data 0.1.0 → 0.3.0

data/lib/test_data/custom_loaders/abstract_base.rb

@@ -0,0 +1,25 @@
+module TestData
+  module CustomLoaders
+    class AbstractBase
+      def name
+        raise Error.new("#name must be defined by CustomLoader subclass")
+      end
+
+      def load_requested(**options)
+      end
+
+      def loaded?(**options)
+        # Check to see if the requested data is already loaded (if possible and
+        # detectable)
+        #
+        # Return true to prevent #load from being called, potentially avoiding an
+        # expensive operation
+        false
+      end
+
+      def load(**options)
+        raise Error.new("#load must be defined by CustomLoader subclass")
+      end
+    end
+  end
+end

data/lib/test_data/custom_loaders/rails_fixtures.rb

@@ -0,0 +1,45 @@
+module TestData
+  module CustomLoaders
+    class RailsFixtures < AbstractBase
+      def initialize
+        @config = TestData.config
+        @statistics = TestData.statistics
+        @already_loaded_rails_fixtures = {}
+      end
+
+      def name
+        :rails_fixtures
+      end
+
+      def validate!(test_instance:)
+        if !test_instance.respond_to?(:setup_fixtures)
+          raise Error.new("'TestData.uses_rails_fixtures(self)' must be passed a test instance that has had ActiveRecord::TestFixtures mixed-in (e.g. `TestData.uses_rails_fixtures(self)` in an ActiveSupport::TestCase `setup` block), but the provided argument does not respond to 'setup_fixtures'")
+        elsif !test_instance.respond_to?(:__test_data_gem_setup_fixtures)
+          raise Error.new("'TestData.uses_rails_fixtures(self)' depends on Rails' default fixture-loading behavior being disabled by calling 'TestData.prevent_rails_fixtures_from_loading_automatically!' as early as possible (e.g. near the top of your test_helper.rb), but it looks like it was never called.")
+        end
+      end
+
+      def load_requested(test_instance:)
+        ActiveRecord::FixtureSet.reset_cache
+        test_instance.instance_variable_set(:@loaded_fixtures,
+          @already_loaded_rails_fixtures.slice(*test_instance.class.fixture_table_names))
+        test_instance.instance_variable_set(:@fixture_cache, {})
+      end
+
+      def loaded?(test_instance:)
+        test_instance.class.fixture_table_names.all? { |table_name|
+          @already_loaded_rails_fixtures.key?(table_name)
+        }
+      end
+
+      def load(test_instance:)
+        test_instance.pre_loaded_fixtures = false
+        test_instance.use_transactional_tests = false
+        test_instance.__test_data_gem_setup_fixtures
+        @already_loaded_rails_fixtures = test_instance.instance_variable_get(:@loaded_fixtures)
+        @statistics.count_load_rails_fixtures!
+        @config.after_rails_fixture_load_hook.call
+      end
+    end
+  end
+end

data/lib/test_data/detects_database_existence.rb

@@ -0,0 +1,19 @@
+module TestData
+  class DetectsDatabaseExistence
+    def initialize
+      @config = TestData.config
+    end
+
+    def call
+      rows = ActiveRecord::Base.connection.execute <<~SQL
+        select datname database_name
+        from pg_catalog.pg_database
+      SQL
+      rows.any? { |row|
+        row["database_name"] == @config.database_name
+      }
+    rescue ActiveRecord::NoDatabaseError
+      false
+    end
+  end
+end

data/lib/test_data/determines_databases_associated_dump_time.rb

@@ -0,0 +1,13 @@
+module TestData
+  class DeterminesDatabasesAssociatedDumpTime
+    def call
+      if (last_dumped_at = ActiveRecord::InternalMetadata.find_by(key: "test_data:last_dumped_at")&.value)
+        Time.parse(last_dumped_at)
+      end
+    rescue ActiveRecord::StatementInvalid
+      # This will be raised if the DB exists but hasn't been migrated/schema-loaded
+    rescue ActiveRecord::NoDatabaseError
+      # This will be raised if the DB doesn't exist yet, which we don't need to warn about
+    end
+  end
+end

data/lib/test_data/determines_when_sql_dump_was_made.rb

@@ -0,0 +1,24 @@
+module TestData
+  class DeterminesWhenSqlDumpWasMade
+    def initialize
+      @config = TestData.config
+    end
+
+    def call
+      if (last_dumped_at = find_last_dumped_value)
+        Time.zone.parse(last_dumped_at)
+      end
+    end
+
+    private
+
+    def find_last_dumped_value
+      return unless File.exist?(@config.non_test_data_dump_path)
+      File.open(@config.non_test_data_dump_path, "r").each_line do |line|
+        if (match = line.match(/INSERT INTO public\.ar_internal_metadata VALUES \('test_data:last_dumped_at', '([^']*)'/))
+          return match[1]
+        end
+      end
+    end
+  end
+end

data/lib/test_data/dumps_database.rb

@@ -6,9 +6,12 @@ module TestData
   class DumpsDatabase
     def initialize
       @config = TestData.config
+      @records_dump_metadata = RecordsDumpMetadata.new
     end
 
     def call
+      @records_dump_metadata.call
+
       dump(
         type: :schema,
         database_name: @config.database_name,

data/lib/test_data/inserts_test_data.rb

@@ -0,0 +1,25 @@
+module TestData
+  class InsertsTestData
+    def initialize
+      @config = TestData.config
+      @statistics = TestData.statistics
+    end
+
+    def call
+      search_path = connection.execute("show search_path").first["search_path"]
+      connection.disable_referential_integrity do
+        connection.execute(File.read(@config.data_dump_full_path))
+      end
+      connection.execute <<~SQL
+        select pg_catalog.set_config('search_path', '#{search_path}', false)
+      SQL
+      @statistics.count_load!
+    end
+
+    private
+
+    def connection
+      ActiveRecord::Base.connection
+    end
+  end
+end

data/lib/test_data/manager.rb

@@ -0,0 +1,187 @@
+module TestData
+  class Manager
+    def initialize
+      @inserts_test_data = InsertsTestData.new
+      @truncates_test_data = TruncatesTestData.new
+      @config = TestData.config
+      @statistics = TestData.statistics
+      @save_points = []
+    end
+
+    def load
+      ensure_after_load_save_point_is_active_if_data_is_loaded!
+      return rollback_to_after_data_load if save_point_active?(:after_data_load)
+
+      create_save_point(:before_data_load)
+      @inserts_test_data.call
+      @config.after_test_data_load_hook.call
+      record_ar_internal_metadata_that_test_data_is_loaded
+      create_save_point(:after_data_load)
+    end
+
+    def truncate
+      ensure_after_load_save_point_is_active_if_data_is_loaded!
+      ensure_after_truncate_save_point_is_active_if_data_is_truncated!
+      return rollback_to_after_data_truncate if save_point_active?(:after_data_truncate)
+
+      if save_point_active?(:after_data_load)
+        # If a test that uses the test data runs before a test that starts by
+        # calling truncate, tables in the database that would NOT be truncated
+        # may have been changed. To avoid this category of test pollution, start
+        # the truncation by rolling back to the known clean point
+        rollback_to_after_data_load
+      else
+        # Seems silly loading data when the user asked us to truncate, but
+        # it's important that the state of the transaction stack matches the
+        # mental model we advertise, because any _other_ test in their suite
+        # should expect that the existence of :after_data_truncate save point
+        # implies that it's safe to rollback to the :after_data_load save
+        # point; since tests run in random order, it's likely to happen
+        TestData.log.debug("TestData.uses_clean_slate was called, but data was not loaded. Loading data before truncate to preserve the transaction save point ordering")
+        load
+      end
+
+      @truncates_test_data.call
+      @config.after_test_data_truncate_hook.call
+      record_ar_internal_metadata_that_test_data_is_truncated
+      create_save_point(:after_data_truncate)
+    end
+
+    def load_custom_data(loader, **options)
+      loader.validate!(**options)
+      snapshot_name = "user_#{loader.name}".to_sym
+
+      ensure_after_load_save_point_is_active_if_data_is_loaded!
+      ensure_after_truncate_save_point_is_active_if_data_is_truncated!
+      ensure_custom_save_point_is_active_if_memo_exists!(snapshot_name)
+
+      loader.load_requested(**options)
+      if save_point_active?(snapshot_name) && loader.loaded?(**options)
+        return rollback_to_custom_savepoint(snapshot_name)
+      end
+
+      if save_point_active?(:after_data_truncate)
+        rollback_to_after_data_truncate
+      else
+        truncate
+      end
+
+      loader.load(**options)
+      record_ar_internal_metadata_of_custom_save_point(snapshot_name)
+      create_save_point(snapshot_name)
+    end
+
+    def rollback_to_before_data_load
+      if save_point_active?(:before_data_load)
+        rollback_save_point(:before_data_load)
+        # No need to recreate the save point
+        # (TestData.uses_test_data will if called)
+      end
+    end
+
+    private
+
+    def rollback_to_after_data_load
+      if save_point_active?(:after_data_load)
+        rollback_save_point(:after_data_load)
+        create_save_point(:after_data_load)
+      end
+    end
+
+    def rollback_to_after_data_truncate
+      if save_point_active?(:after_data_truncate)
+        rollback_save_point(:after_data_truncate)
+        create_save_point(:after_data_truncate)
+      end
+    end
+
+    def rollback_to_custom_savepoint(name)
+      if save_point_active?(name)
+        rollback_save_point(name)
+        create_save_point(name)
+      end
+    end
+
+    def connection
+      ActiveRecord::Base.connection
+    end
+
+    def ensure_after_load_save_point_is_active_if_data_is_loaded!
+      if !save_point_active?(:after_data_load) && ar_internal_metadata_shows_test_data_is_loaded?
+        TestData.log.debug "Test data appears to be loaded, but the :after_data_load save point was rolled back (and not by this gem). Recreating the :after_data_load save point"
+        create_save_point(:after_data_load)
+      end
+    end
+
+    def record_ar_internal_metadata_that_test_data_is_loaded
+      if ar_internal_metadata_shows_test_data_is_loaded?
+        TestData.log.warn "Attempted to record that test data is loaded in ar_internal_metadata, but record already existed. Perhaps a previous test run committed your test data?"
+      else
+        ActiveRecord::InternalMetadata.create!(key: "test_data:loaded", value: "true")
+      end
+    end
+
+    def ar_internal_metadata_shows_test_data_is_loaded?
+      ActiveRecord::InternalMetadata.find_by(key: "test_data:loaded")&.value == "true"
+    end
+
+    def ensure_after_truncate_save_point_is_active_if_data_is_truncated!
+      if !save_point_active?(:after_data_truncate) && ar_internal_metadata_shows_test_data_is_truncated?
+        TestData.log.debug "Test data appears to be loaded, but the :after_data_truncate save point was rolled back (and not by this gem). Recreating the :after_data_truncate save point"
+        create_save_point(:after_data_truncate)
+      end
+    end
+
+    def record_ar_internal_metadata_that_test_data_is_truncated
+      if ar_internal_metadata_shows_test_data_is_truncated?
+        TestData.log.warn "Attempted to record that test data is truncated in ar_internal_metadata, but record already existed. Perhaps a previous test run committed the truncation of your test data?"
+      else
+        ActiveRecord::InternalMetadata.create!(key: "test_data:truncated", value: "true")
+      end
+    end
+
+    def ar_internal_metadata_shows_test_data_is_truncated?
+      ActiveRecord::InternalMetadata.find_by(key: "test_data:truncated")&.value == "true"
+    end
+
+    def ensure_custom_save_point_is_active_if_memo_exists!(name)
+      if !save_point_active?(name) && ar_internal_metadata_shows_custom_operation_was_persisted?(name)
+        TestData.log.debug "#{name} appears to have been loaded by test_data, but the #{name} save point was rolled back (and not by this gem). Recreating the #{name} save point"
+        create_save_point(name)
+      end
+    end
+
+    def ar_internal_metadata_shows_custom_operation_was_persisted?(name)
+      ActiveRecord::InternalMetadata.find_by(key: "test_data:#{name}")&.value == "true"
+    end
+
+    def record_ar_internal_metadata_of_custom_save_point(name)
+      if ar_internal_metadata_shows_custom_operation_was_persisted?(name)
+        TestData.log.warn "Attempted to record that test_data had loaded #{name} in ar_internal_metadata, but record already existed. Perhaps a previous test run committed it?"
+      else
+        ActiveRecord::InternalMetadata.create!(key: "test_data:#{name}", value: "true")
+      end
+    end
+
+    def save_point_active?(name)
+      purge_closed_save_points!
+      !!@save_points.find { |sp| sp.name == name }&.active?
+    end
+
+    def create_save_point(name)
+      raise Error.new("Could not create test_data savepoint '#{name}', because it was already active!") if save_point_active?(name)
+      @save_points << SavePoint.new(name)
+    end
+
+    def rollback_save_point(name)
+      @save_points.find { |sp| sp.name == name }&.rollback!
+      purge_closed_save_points!
+    end
+
+    def purge_closed_save_points!
+      @save_points = @save_points.select { |save_point|
+        save_point.active?
+      }
+    end
+  end
+end

data/lib/test_data/railtie.rb

@@ -8,5 +8,9 @@ module TestData
     rake_tasks do
       load Pathname.new(__dir__).join("rake.rb")
     end
+
+    initializer "test_data.validate_data_up_to_date" do
+      WarnsIfDumpIsNewerThanDatabase.new.call
+    end
   end
 end

data/lib/test_data/rake.rb

@@ -10,17 +10,18 @@ def run_in_test_data_env(task_name)
 end
 
 def create_database_or_else_blow_up_if_its_not_empty!
+  raise unless Rails.env.test_data?
+
+  unless TestData::DetectsDatabaseExistence.new.call
+    Rake::Task["test_data:create_database"].invoke
+  end
+
   unless TestData::DetectsDatabaseEmptiness.new.empty?
     raise TestData::Error.new("Database '#{TestData.config.database_name}' already exists and is not empty. To re-initialize it, drop it first (e.g. `rake test_data:drop_database`)")
   end
-rescue TestData::Error => e
-  raise e
-rescue
-  # Only (anticipated) cause for raise here is DB did not exist
-  Rake::Task["test_data:create_database"].invoke
 end
 
-desc "Verifies test_data environment looks good"
+desc "Verifies that the test_data environment looks good"
 task "test_data:verify_config" do
   TestData.log.with_plain_writer do
     config = TestData::VerifiesConfiguration.new.call
@@ -34,12 +35,12 @@ task "test_data:verify_config" do
   end
 end
 
-desc "Install default configuration files and snippets"
+desc "Installs default configuration files and snippets"
 task "test_data:configure" do
   TestData::InstallsConfiguration.new.call
 end
 
-desc "Initialize test_data's interactive database"
+desc "Initializes test_data's interactive database"
 task "test_data:initialize" => ["test_data:verify_config", :environment] do
   next run_in_test_data_env("test_data:initialize") if wrong_env?
 
@@ -52,15 +53,43 @@ task "test_data:initialize" => ["test_data:verify_config", :environment] do
   end
 
   TestData.log.info <<~MSG
-    Your test_data environment and database are ready for use! You can now run
-    your server (or any command) to create some test data like so:
+    Your test_data environment and database are ready for use! You can now run your server (or any command) to create some test data like so:
 
       $ RAILS_ENV=test_data bin/rails server
 
   MSG
 end
 
-desc "Initialize test_data Rails environment & configure database"
+desc "Re-initializes test_data's interactive database (by dropping and reloading it)"
+task "test_data:reinitialize" => ["test_data:verify_config", :environment] do
+  next run_in_test_data_env("test_data:reinitialize") if wrong_env?
+
+  # Take caution only if the test_data database exists
+  if TestData::DetectsDatabaseExistence.new.call
+    TestData::WarnsIfDatabaseIsNewerThanDump.new.call
+
+    unless ENV["TEST_DATA_CONFIRM"].present?
+      confirmed = if $stdin.isatty
+        puts "This will DROP test_data database '#{TestData.config.database_name}'. Are you sure you want to re-initialize it? [yN]"
+        $stdin.gets.chomp.downcase.start_with?("y")
+      else
+        puts "'#{TestData.config.database_name}' exists. Set TEST_DATA_CONFIRM=true to drop the database and re-initialize it."
+        false
+      end
+
+      unless confirmed
+        puts "Exiting without making any changes"
+        exit 1
+      end
+    end
+
+    Rake::Task["test_data:drop_database"].invoke
+  end
+
+  Rake::Task["test_data:initialize"].invoke
+end
+
+desc "Initializes test_data Rails environment & configure database"
 task "test_data:install" => ["test_data:configure", "test_data:initialize"]
 
 desc "Dumps the interactive test_data database"
@@ -83,7 +112,7 @@ task "test_data:load" => ["test_data:verify_config", :environment] do
   TestData::LoadsDatabaseDumps.new.call
 
   if ActiveRecord::Base.connection.migration_context.needs_migration?
-    TestData.log.warn "There are pending migrations for database '#{TestData.config.database_name}'. To run them, run:\n\n  RAILS_ENV=test_data bin/rake db:migrate\n\n"
+    TestData.log.warn "There are pending migrations for database '#{TestData.config.database_name}'. To run them, run:\n\n  $ RAILS_ENV=test_data bin/rake db:migrate\n\n"
   end
 end