test_data 0.0.2 → 0.2.2

data/lib/test_data/custom_loaders/abstract_base.rb

@@ -0,0 +1,25 @@
+module TestData
+  module CustomLoaders
+    class AbstractBase
+      def name
+        raise Error.new("#name must be defined by CustomLoader subclass")
+      end
+
+      def load_requested(**options)
+      end
+
+      def loaded?(**options)
+        # Check to see if the requested data is already loaded (if possible and
+        # detectable)
+        #
+        # Return true to prevent #load from being called, potentially avoiding an
+        # expensive operation
+        false
+      end
+
+      def load(**options)
+        raise Error.new("#load must be defined by CustomLoader subclass")
+      end
+    end
+  end
+end

data/lib/test_data/custom_loaders/rails_fixtures.rb

@@ -0,0 +1,45 @@
+module TestData
+  module CustomLoaders
+    class RailsFixtures < AbstractBase
+      def initialize
+        @config = TestData.config
+        @statistics = TestData.statistics
+        @already_loaded_rails_fixtures = {}
+      end
+
+      def name
+        :rails_fixtures
+      end
+
+      def validate!(test_instance:)
+        if !test_instance.respond_to?(:setup_fixtures)
+          raise Error.new("'TestData.uses_rails_fixtures(self)' must be passed a test instance that has had ActiveRecord::TestFixtures mixed-in (e.g. `TestData.uses_rails_fixtures(self)` in an ActiveSupport::TestCase `setup` block), but the provided argument does not respond to 'setup_fixtures'")
+        elsif !test_instance.respond_to?(:__test_data_gem_setup_fixtures)
+          raise Error.new("'TestData.uses_rails_fixtures(self)' depends on Rails' default fixture-loading behavior being disabled by calling 'TestData.prevent_rails_fixtures_from_loading_automatically!' as early as possible (e.g. near the top of your test_helper.rb), but it looks like it was never called.")
+        end
+      end
+
+      def load_requested(test_instance:)
+        ActiveRecord::FixtureSet.reset_cache
+        test_instance.instance_variable_set(:@loaded_fixtures,
+          @already_loaded_rails_fixtures.slice(*test_instance.class.fixture_table_names))
+        test_instance.instance_variable_set(:@fixture_cache, {})
+      end
+
+      def loaded?(test_instance:)
+        test_instance.class.fixture_table_names.all? { |table_name|
+          @already_loaded_rails_fixtures.key?(table_name)
+        }
+      end
+
+      def load(test_instance:)
+        test_instance.pre_loaded_fixtures = false
+        test_instance.use_transactional_tests = false
+        test_instance.__test_data_gem_setup_fixtures
+        @already_loaded_rails_fixtures = test_instance.instance_variable_get(:@loaded_fixtures)
+        @statistics.count_load_rails_fixtures!
+        @config.after_rails_fixture_load_hook.call
+      end
+    end
+  end
+end

data/lib/test_data/dumps_database.rb

@@ -40,12 +40,14 @@ module TestData
     def dump(type:, database_name:, relative_path:, full_path:, name: type, flags: "")
       dump_pathname = Pathname.new(full_path)
       FileUtils.mkdir_p(File.dirname(dump_pathname))
+      before_size = File.size?(dump_pathname)
       if execute("pg_dump #{database_name} --no-tablespaces --no-owner --inserts --#{type}-only #{flags} -f #{dump_pathname}")
         prepend_set_replication_role!(full_path) if type == :data
 
         TestData.log.info "Dumped '#{database_name}' #{name} to '#{relative_path}'"
+        log_size_info_and_warnings(before_size: before_size, after_size: File.size(dump_pathname))
       else
-        raise "Failed while attempting to  dump '#{database_name}' #{name} to '#{relative_path}'"
+        raise Error.new("Failed while attempting to  dump '#{database_name}' #{name} to '#{relative_path}'")
       end
     end
 
@@ -72,5 +74,26 @@ module TestData
       COMMAND
       TestData.log.debug("Prepended replication role instruction to '#{data_dump_path}'")
     end
+
+    def log_size_info_and_warnings(before_size:, after_size:)
+      percent_change = percent_change(before_size, after_size)
+      TestData.log.info "  Size: #{to_size(after_size)}#{" (#{percent_change}% #{before_size > after_size ? "decrease" : "increase"})" if percent_change}"
+      if after_size > 5242880
+        TestData.log.warn "  WARNING: file size exceeds 5MB. Be sure to only persist what data you need to sufficiently test your application"
+      end
+      if before_size && (after_size - before_size) > 1048576
+        TestData.log.warn "  WARNING: size of this dump increased by #{to_size(after_size - before_size)}. You may want to inspect the file to validate extraneous data was not committed"
+      end
+    end
+
+    def percent_change(before_size, after_size)
+      return unless before_size && before_size > 0 && after_size
+      ((before_size - after_size).abs / before_size * 100).round(2)
+    end
+
+    def to_size(bytes)
+      e = Math.log10(bytes).to_i / 3
+      "%.0f" % (bytes / 1000**e) + [" bytes", "KB", "MB", "GB"][e]
+    end
   end
 end

data/lib/test_data/generator_support.rb

@@ -0,0 +1,3 @@
+module TestData
+  BEFORE_TEST_STANZA_REGEX = /^$\n(?:^\#.*\n)*^test:/
+end

data/lib/test_data/inserts_test_data.rb

@@ -0,0 +1,25 @@
+module TestData
+  class InsertsTestData
+    def initialize
+      @config = TestData.config
+      @statistics = TestData.statistics
+    end
+
+    def call
+      search_path = connection.execute("show search_path").first["search_path"]
+      connection.disable_referential_integrity do
+        connection.execute(File.read(@config.data_dump_full_path))
+      end
+      connection.execute <<~SQL
+        select pg_catalog.set_config('search_path', '#{search_path}', false)
+      SQL
+      @statistics.count_load!
+    end
+
+    private
+
+    def connection
+      ActiveRecord::Base.connection
+    end
+  end
+end

data/lib/test_data/loads_database_dumps.rb

@@ -38,7 +38,7 @@ module TestData
       if system "psql -q -d #{database_name} < #{dump_pathname}"
         TestData.log.info "Loaded #{name} from '#{relative_path}' into database '#{database_name}' "
       else
-        raise "Failed while attempting to load #{name} from '#{relative_path}' into database '#{database_name}'"
+        raise Error.new("Failed while attempting to load #{name} from '#{relative_path}' into database '#{database_name}'")
       end
     end
   end

data/lib/test_data/log.rb

@@ -6,13 +6,20 @@ module TestData
   class Log
     LEVELS = [:debug, :info, :warn, :error, :quiet]
     DEFAULT_WRITER = ->(message, level) do
-      output = "[test_data: #{level}] #{message}"
+      output = "[test_data:#{level}] #{message}"
       if [:warn, :error].include?(level)
         warn output
       else
         puts output
       end
     end
+    PLAIN_WRITER = ->(message, level) do
+      if [:warn, :error].include?(level)
+        warn message
+      else
+        puts message
+      end
+    end
 
     attr_reader :level, :writer
 
@@ -49,6 +56,17 @@ module TestData
       end
     end
 
+    def with_writer(writer, &blk)
+      og_writer = self.writer
+      self.writer = writer
+      blk.call
+      self.writer = og_writer
+    end
+
+    def with_plain_writer(&blk)
+      with_writer(PLAIN_WRITER, &blk)
+    end
+
     private
 
     def enabled?(level)

data/lib/test_data/{transactional_data_loader.rb → manager.rb}

@@ -1,69 +1,25 @@
 module TestData
-  def self.load(transactions: true)
-    @transactional_data_loader ||= TransactionalDataLoader.new
-    @transactional_data_loader.load(transactions: transactions)
-  end
-
-  def self.rollback(save_point_name = :after_data_load)
-    @transactional_data_loader ||= TransactionalDataLoader.new
-    case save_point_name
-    when :before_data_load
-      @transactional_data_loader.rollback_to_before_data_load
-    when :after_data_load
-      @transactional_data_loader.rollback_to_after_data_load
-    when :after_data_truncate
-      @transactional_data_loader.rollback_to_after_data_truncate
-    else
-      raise Error.new("No known save point named '#{save_point_name}'. Valid values are: [:before_data_load, :after_data_load, :after_data_truncate]")
-    end
-  end
-
-  def self.truncate(transactions: true)
-    @transactional_data_loader ||= TransactionalDataLoader.new
-    @transactional_data_loader.truncate(transactions: transactions)
-  end
-
-  class TransactionalDataLoader
+  class Manager
     def initialize
+      @inserts_test_data = InsertsTestData.new
+      @truncates_test_data = TruncatesTestData.new
       @config = TestData.config
       @statistics = TestData.statistics
       @save_points = []
     end
 
-    def load(transactions: true)
-      return execute_data_load unless transactions
+    def load
       ensure_after_load_save_point_is_active_if_data_is_loaded!
       return rollback_to_after_data_load if save_point_active?(:after_data_load)
 
       create_save_point(:before_data_load)
-      execute_data_load
+      @inserts_test_data.call
+      @config.after_test_data_load_hook.call
       record_ar_internal_metadata_that_test_data_is_loaded
       create_save_point(:after_data_load)
     end
 
-    def rollback_to_before_data_load
-      if save_point_active?(:before_data_load)
-        rollback_save_point(:before_data_load)
-        # No need to recreate the save point -- TestData.load will if called
-      end
-    end
-
-    def rollback_to_after_data_load
-      if save_point_active?(:after_data_load)
-        rollback_save_point(:after_data_load)
-        create_save_point(:after_data_load)
-      end
-    end
-
-    def rollback_to_after_data_truncate
-      if save_point_active?(:after_data_truncate)
-        rollback_save_point(:after_data_truncate)
-        create_save_point(:after_data_truncate)
-      end
-    end
-
-    def truncate(transactions: true)
-      return execute_data_truncate unless transactions
+    def truncate
       ensure_after_load_save_point_is_active_if_data_is_loaded!
       ensure_after_truncate_save_point_is_active_if_data_is_truncated!
       return rollback_to_after_data_truncate if save_point_active?(:after_data_truncate)
@@ -81,17 +37,75 @@ module TestData
         # should expect that the existence of :after_data_truncate save point
         # implies that it's safe to rollback to the :after_data_load save
         # point; since tests run in random order, it's likely to happen
-        TestData.log.debug("TestData.truncate was called, but data was not loaded. Loading data before truncate to preserve the documents transaction save point ordering")
-        load(transactions: true)
+        TestData.log.debug("TestData.uses_clean_slate was called, but data was not loaded. Loading data before truncate to preserve the transaction save point ordering")
+        load
       end
 
-      execute_data_truncate
+      @truncates_test_data.call
+      @config.after_test_data_truncate_hook.call
       record_ar_internal_metadata_that_test_data_is_truncated
       create_save_point(:after_data_truncate)
     end
 
+    def load_custom_data(loader, **options)
+      loader.validate!(**options)
+      snapshot_name = "user_#{loader.name}".to_sym
+
+      ensure_after_load_save_point_is_active_if_data_is_loaded!
+      ensure_after_truncate_save_point_is_active_if_data_is_truncated!
+      ensure_custom_save_point_is_active_if_memo_exists!(snapshot_name)
+
+      loader.load_requested(**options)
+      if save_point_active?(snapshot_name) && loader.loaded?(**options)
+        return rollback_to_custom_savepoint(snapshot_name)
+      end
+
+      if save_point_active?(:after_data_truncate)
+        rollback_to_after_data_truncate
+      else
+        truncate
+      end
+
+      loader.load(**options)
+      record_ar_internal_metadata_of_custom_save_point(snapshot_name)
+      create_save_point(snapshot_name)
+    end
+
+    def rollback_to_before_data_load
+      if save_point_active?(:before_data_load)
+        rollback_save_point(:before_data_load)
+        # No need to recreate the save point
+        # (TestData.uses_test_data will if called)
+      end
+    end
+
     private
 
+    def rollback_to_after_data_load
+      if save_point_active?(:after_data_load)
+        rollback_save_point(:after_data_load)
+        create_save_point(:after_data_load)
+      end
+    end
+
+    def rollback_to_after_data_truncate
+      if save_point_active?(:after_data_truncate)
+        rollback_save_point(:after_data_truncate)
+        create_save_point(:after_data_truncate)
+      end
+    end
+
+    def rollback_to_custom_savepoint(name)
+      if save_point_active?(name)
+        rollback_save_point(name)
+        create_save_point(name)
+      end
+    end
+
+    def connection
+      ActiveRecord::Base.connection
+    end
+
     def ensure_after_load_save_point_is_active_if_data_is_loaded!
       if !save_point_active?(:after_data_load) && ar_internal_metadata_shows_test_data_is_loaded?
         TestData.log.debug "Test data appears to be loaded, but the :after_data_load save point was rolled back (and not by this gem). Recreating the :after_data_load save point"
@@ -130,31 +144,22 @@ module TestData
       ActiveRecord::InternalMetadata.find_by(key: "test_data:truncated")&.value == "true"
     end
 
-    def execute_data_load
-      search_path = execute("show search_path").first["search_path"]
-      connection.disable_referential_integrity do
-        execute(File.read(@config.data_dump_full_path))
+    def ensure_custom_save_point_is_active_if_memo_exists!(name)
+      if !save_point_active?(name) && ar_internal_metadata_shows_custom_operation_was_persisted?(name)
+        TestData.log.debug "#{name} appears to have been loaded by test_data, but the #{name} save point was rolled back (and not by this gem). Recreating the #{name} save point"
+        create_save_point(name)
       end
-      execute <<~SQL
-        select pg_catalog.set_config('search_path', '#{search_path}', false)
-      SQL
-      @statistics.count_load!
     end
 
-    def execute_data_truncate
-      connection.disable_referential_integrity do
-        execute("TRUNCATE TABLE #{tables_to_truncate.map { |t| connection.quote_table_name(t) }.join(", ")}")
-      end
-      @statistics.count_truncate!
+    def ar_internal_metadata_shows_custom_operation_was_persisted?(name)
+      ActiveRecord::InternalMetadata.find_by(key: "test_data:#{name}")&.value == "true"
     end
 
-    def tables_to_truncate
-      if @config.truncate_these_test_data_tables.present?
-        @config.truncate_these_test_data_tables
+    def record_ar_internal_metadata_of_custom_save_point(name)
+      if ar_internal_metadata_shows_custom_operation_was_persisted?(name)
+        TestData.log.warn "Attempted to record that test_data had loaded #{name} in ar_internal_metadata, but record already existed. Perhaps a previous test run committed it?"
       else
-        @tables_to_truncate ||= IO.foreach(@config.data_dump_path).grep(/^INSERT INTO/) { |line|
-          line.match(/^INSERT INTO ([^\s]+)/)&.captures&.first
-        }.compact.uniq
+        ActiveRecord::InternalMetadata.create!(key: "test_data:#{name}", value: "true")
       end
     end
 
@@ -178,13 +183,5 @@ module TestData
         save_point.active?
       }
     end
-
-    def execute(sql)
-      connection.execute(sql)
-    end
-
-    def connection
-      ActiveRecord::Base.connection
-    end
   end
 end

data/lib/test_data/rake.rb

@@ -22,13 +22,15 @@ end
 
 desc "Verifies test_data environment looks good"
 task "test_data:verify_config" do
-  config = TestData::VerifiesConfiguration.new.call
-  unless config.looks_good?
-    TestData.log.warn "The test_data gem is not configured correctly. Try 'rake test_data:configure'?\n"
-    config.problems.each do |problem|
-      TestData.log.warn "  - #{problem}"
+  TestData.log.with_plain_writer do
+    config = TestData::VerifiesConfiguration.new.call
+    unless config.looks_good?
+      TestData.log.warn "\nThe test_data gem is not configured correctly. Try running: rake test_data:configure\n\n"
+      config.problems.each do |problem|
+        TestData.log.warn "  - #{problem}"
+      end
+      fail
     end
-    fail
   end
 end
 
@@ -48,6 +50,13 @@ task "test_data:initialize" => ["test_data:verify_config", :environment] do
     ActiveRecord::Tasks::DatabaseTasks.load_schema_current(ActiveRecord::Base.schema_format, ENV["SCHEMA"], "test_data")
     ActiveRecord::Tasks::DatabaseTasks.load_seed
   end
+
+  TestData.log.info <<~MSG
+    Your test_data environment and database are ready for use! You can now run your server (or any command) to create some test data like so:
+
+      $ RAILS_ENV=test_data bin/rails server
+
+  MSG
 end
 
 desc "Initialize test_data Rails environment & configure database"
@@ -73,7 +82,7 @@ task "test_data:load" => ["test_data:verify_config", :environment] do
   TestData::LoadsDatabaseDumps.new.call
 
   if ActiveRecord::Base.connection.migration_context.needs_migration?
-    TestData.log.warn "There are pending migrations for database '#{TestData.config.database_name}'. To run them, run:\n\n  RAILS_ENV=test_data bin/rake db:migrate\n\n"
+    TestData.log.warn "There are pending migrations for database '#{TestData.config.database_name}'. To run them, run:\n\n  $ RAILS_ENV=test_data bin/rake db:migrate\n\n"
   end
 end