test_data 0.0.1 → 0.2.1

data/lib/test_data.rb

@@ -1,20 +1,61 @@
+require_relative "test_data/active_record_ext"
 require_relative "test_data/active_support_ext"
 require_relative "test_data/config"
 require_relative "test_data/configuration_verification"
 require_relative "test_data/configurators"
 require_relative "test_data/configurators/environment_file"
+require_relative "test_data/configurators/initializer"
+require_relative "test_data/configurators/cable_yaml"
 require_relative "test_data/configurators/database_yaml"
+require_relative "test_data/configurators/secrets_yaml"
 require_relative "test_data/configurators/webpacker_yaml"
+require_relative "test_data/custom_loaders/abstract_base"
+require_relative "test_data/custom_loaders/rails_fixtures"
 require_relative "test_data/detects_database_emptiness"
 require_relative "test_data/dumps_database"
 require_relative "test_data/error"
+require_relative "test_data/inserts_test_data"
 require_relative "test_data/installs_configuration"
 require_relative "test_data/loads_database_dumps"
+require_relative "test_data/log"
 require_relative "test_data/railtie"
-require_relative "test_data/transactional_data_loader"
+require_relative "test_data/save_point"
+require_relative "test_data/statistics"
+require_relative "test_data/manager"
+require_relative "test_data/truncates_test_data"
 require_relative "test_data/verifies_configuration"
 require_relative "test_data/verifies_dumps_are_loadable"
 require_relative "test_data/version"
 require_relative "generators/test_data/environment_file_generator"
+require_relative "generators/test_data/initializer_generator"
+require_relative "generators/test_data/cable_yaml_generator"
 require_relative "generators/test_data/database_yaml_generator"
+require_relative "generators/test_data/secrets_yaml_generator"
 require_relative "generators/test_data/webpacker_yaml_generator"
+
+module TestData
+  def self.uninitialize
+    @manager ||= Manager.new
+    @manager.rollback_to_before_data_load
+  end
+
+  def self.uses_test_data
+    @manager ||= Manager.new
+    @manager.load
+  end
+
+  def self.uses_clean_slate
+    @manager ||= Manager.new
+    @manager.truncate
+  end
+
+  def self.uses_rails_fixtures(test_instance)
+    @rails_fixtures_loader ||= CustomLoaders::RailsFixtures.new
+    @manager ||= Manager.new
+    @manager.load_custom_data(@rails_fixtures_loader, test_instance: test_instance)
+  end
+
+  def self.insert_test_data_dump
+    InsertsTestData.new.call
+  end
+end

data/lib/test_data/active_record_ext.rb

@@ -0,0 +1,11 @@
+module TestData
+  def self.prevent_rails_fixtures_from_loading_automatically!
+    ActiveRecord::TestFixtures.define_method(:__test_data_gem_setup_fixtures,
+      ActiveRecord::TestFixtures.instance_method(:setup_fixtures))
+    ActiveRecord::TestFixtures.remove_method(:setup_fixtures)
+    ActiveRecord::TestFixtures.define_method(:setup_fixtures, ->(config = nil) {})
+
+    ActiveRecord::TestFixtures.remove_method(:teardown_fixtures)
+    ActiveRecord::TestFixtures.define_method(:teardown_fixtures, -> {})
+  end
+end

data/lib/test_data/config.rb

@@ -18,9 +18,31 @@ module TestData
     attr_accessor :non_test_data_dump_path
 
     # Tables to exclude from test data dumps
-    attr_accessor :non_test_data_tables
+    attr_writer :non_test_data_tables
+    def non_test_data_tables
+      (@non_test_data_tables + [
+        ActiveRecord::Base.connection.schema_migration.table_name,
+        ActiveRecord::InternalMetadata.table_name
+      ]).uniq
+    end
+
+    # Tables to exclude from all dumps
+    attr_accessor :dont_dump_these_tables
+
+    # Tables to truncate when TestData.uses_clean_slate is called
+    attr_accessor :truncate_these_test_data_tables
+
+    # Log level (valid values: [:debug, :info, :warn, :error, :quiet])
+    def log_level
+      TestData.log.level
+    end
+
+    def log_level=(level)
+      TestData.log.level = level
+    end
 
-    attr_reader :pwd, :database_yaml_path
+    attr_reader :pwd, :cable_yaml_path, :database_yaml_path, :secrets_yaml_path,
+      :after_test_data_load_hook, :after_test_data_truncate_hook, :after_rails_fixture_load_hook
 
     def self.full_path_reader(*relative_path_readers)
       relative_path_readers.each do |relative_path_reader|
@@ -30,15 +52,46 @@ module TestData
       end
     end
 
-    full_path_reader :schema_dump_path, :data_dump_path, :non_test_data_dump_path, :database_yaml_path
+    full_path_reader :schema_dump_path, :data_dump_path, :non_test_data_dump_path, :cable_yaml_path, :database_yaml_path, :secrets_yaml_path
 
     def initialize(pwd:)
       @pwd = pwd
       @schema_dump_path = "test/support/test_data/schema.sql"
       @data_dump_path = "test/support/test_data/data.sql"
       @non_test_data_dump_path = "test/support/test_data/non_test_data.sql"
+      @cable_yaml_path = "config/cable.yml"
       @database_yaml_path = "config/database.yml"
-      @non_test_data_tables = ["ar_internal_metadata", "schema_migrations"]
+      @secrets_yaml_path = "config/secrets.yml"
+      @non_test_data_tables = []
+      @dont_dump_these_tables = []
+      @truncate_these_test_data_tables = nil
+      @after_test_data_load_hook = -> {}
+      @after_test_data_truncate_hook = -> {}
+      @after_rails_fixture_load_hook = -> {}
+    end
+
+    def after_test_data_load(callable = nil, &blk)
+      hook = callable || blk
+      if !hook.respond_to?(:call)
+        raise Error.new("after_test_data_load must be passed a callable (e.g. a Proc) or called with a block")
+      end
+      @after_test_data_load_hook = hook
+    end
+
+    def after_test_data_truncate(callable = nil, &blk)
+      hook = callable || blk
+      if !hook.respond_to?(:call)
+        raise Error.new("after_test_data_truncate must be passed a callable (e.g. a Proc) or called with a block")
+      end
+      @after_test_data_truncate_hook = hook
+    end
+
+    def after_rails_fixture_load(callable = nil, &blk)
+      hook = callable || blk
+      if !hook.respond_to?(:call)
+        raise Error.new("after_rails_fixture_load must be passed a callable (e.g. a Proc) or called with a block")
+      end
+      @after_rails_fixture_load_hook = hook
     end
 
     def database_yaml

data/lib/test_data/configurators.rb

@@ -3,7 +3,10 @@ module TestData
     def self.all
       [
         EnvironmentFile,
+        Initializer,
+        CableYaml,
         DatabaseYaml,
+        SecretsYaml,
         WebpackerYaml
       ].map(&:new)
     end

data/lib/test_data/configurators/cable_yaml.rb

@@ -0,0 +1,25 @@
+module TestData
+  module Configurators
+    class CableYaml
+      def initialize
+        @generator = CableYamlGenerator.new
+        @config = TestData.config
+      end
+
+      def verify
+        if !File.exist?(@config.cable_yaml_full_path) ||
+            YAML.load_file(@config.cable_yaml_full_path).key?("test_data")
+          ConfigurationVerification.new(looks_good?: true)
+        else
+          ConfigurationVerification.new(problems: [
+            "'#{@config.cable_yaml_path}' exists but does not contain a 'test_data' section"
+          ])
+        end
+      end
+
+      def configure
+        @generator.call
+      end
+    end
+  end
+end

data/lib/test_data/configurators/environment_file.rb

@@ -7,12 +7,13 @@ module TestData
       end
 
       def verify
-        pathname = Pathname.new("#{@config.pwd}/config/environments/test_data.rb")
+        path = "config/environments/test_data.rb"
+        pathname = Pathname.new("#{@config.pwd}/#{path}")
         if pathname.readable?
           ConfigurationVerification.new(looks_good?: true)
         else
           ConfigurationVerification.new(problems: [
-            "'#{pathname}' is not readable"
+            "'#{path}' is not readable"
           ])
         end
       end

data/lib/test_data/configurators/initializer.rb

@@ -0,0 +1,26 @@
+module TestData
+  module Configurators
+    class Initializer
+      def initialize
+        @generator = InitializerGenerator.new
+        @config = TestData.config
+      end
+
+      def verify
+        path = "config/initializers/test_data.rb"
+        pathname = Pathname.new("#{@config.pwd}/#{path}")
+        if pathname.readable?
+          ConfigurationVerification.new(looks_good?: true)
+        else
+          ConfigurationVerification.new(problems: [
+            "'#{path}' is not readable"
+          ])
+        end
+      end
+
+      def configure
+        @generator.call
+      end
+    end
+  end
+end

data/lib/test_data/configurators/secrets_yaml.rb

@@ -0,0 +1,25 @@
+module TestData
+  module Configurators
+    class SecretsYaml
+      def initialize
+        @generator = SecretsYamlGenerator.new
+        @config = TestData.config
+      end
+
+      def verify
+        if !File.exist?(@config.secrets_yaml_full_path) ||
+            YAML.load_file(@config.secrets_yaml_full_path).key?("test_data")
+          ConfigurationVerification.new(looks_good?: true)
+        else
+          ConfigurationVerification.new(problems: [
+            "'#{@config.secrets_yaml_path}' exists but does not contain a 'test_data' section"
+          ])
+        end
+      end
+
+      def configure
+        @generator.call
+      end
+    end
+  end
+end

data/lib/test_data/configurators/webpacker_yaml.rb

@@ -7,16 +7,17 @@ module TestData
       end
 
       def verify
-        pathname = Pathname.new("#{@config.pwd}/config/webpacker.yml")
+        path = "config/webpacker.yml"
+        pathname = Pathname.new("#{@config.pwd}/#{path}")
         return ConfigurationVerification.new(looks_good?: true) unless pathname.readable?
         yaml = load_yaml(pathname)
         if yaml.nil?
           ConfigurationVerification.new(problems: [
-            "'#{pathname}' is not valid YAML"
+            "'#{path}' is not valid YAML"
           ])
         elsif !yaml.key?("test_data")
           ConfigurationVerification.new(problems: [
-            "'#{pathname}' does not contain a 'test_data' section"
+            "'#{path}' does not contain a 'test_data' section"
           ])
         else
           ConfigurationVerification.new(looks_good?: true)

data/lib/test_data/custom_loaders/abstract_base.rb

@@ -0,0 +1,25 @@
+module TestData
+  module CustomLoaders
+    class AbstractBase
+      def name
+        raise Error.new("#name must be defined by CustomLoader subclass")
+      end
+
+      def load_requested(**options)
+      end
+
+      def loaded?(**options)
+        # Check to see if the requested data is already loaded (if possible and
+        # detectable)
+        #
+        # Return true to prevent #load from being called, potentially avoiding an
+        # expensive operation
+        false
+      end
+
+      def load(**options)
+        raise Error.new("#load must be defined by CustomLoader subclass")
+      end
+    end
+  end
+end

data/lib/test_data/custom_loaders/rails_fixtures.rb

@@ -0,0 +1,42 @@
+module TestData
+  module CustomLoaders
+    class RailsFixtures < AbstractBase
+      def initialize
+        @config = TestData.config
+        @statistics = TestData.statistics
+        @already_loaded_rails_fixtures = {}
+      end
+
+      def name
+        :rails_fixtures
+      end
+
+      def validate!(test_instance:)
+        if !test_instance.respond_to?(:setup_fixtures)
+          raise Error.new("'TestData.uses_rails_fixtures(self)' must be passed a test instance that has had ActiveRecord::TestFixtures mixed-in (e.g. `TestData.uses_rails_fixtures(self)` in an ActiveSupport::TestCase `setup` block), but the provided argument does not respond to 'setup_fixtures'")
+        elsif !test_instance.respond_to?(:__test_data_gem_setup_fixtures)
+          raise Error.new("'TestData.uses_rails_fixtures(self)' depends on Rails' default fixture-loading behavior being disabled by calling 'TestData.prevent_rails_fixtures_from_loading_automatically!' as early as possible (e.g. near the top of your test_helper.rb), but it looks like it was never called.")
+        end
+      end
+
+      def load_requested(test_instance:)
+        ActiveRecord::FixtureSet.reset_cache
+        test_instance.instance_variable_set(:@loaded_fixtures, @already_loaded_rails_fixtures[test_instance.class])
+        test_instance.instance_variable_set(:@fixture_cache, {})
+      end
+
+      def loaded?(test_instance:)
+        @already_loaded_rails_fixtures[test_instance.class].present?
+      end
+
+      def load(test_instance:)
+        test_instance.pre_loaded_fixtures = false
+        test_instance.use_transactional_tests = false
+        test_instance.__test_data_gem_setup_fixtures
+        @already_loaded_rails_fixtures[test_instance.class] = test_instance.instance_variable_get(:@loaded_fixtures)
+        @statistics.count_load_rails_fixtures!
+        @config.after_rails_fixture_load_hook.call
+      end
+    end
+  end
+end

data/lib/test_data/dumps_database.rb

@@ -1,5 +1,6 @@
 require "pathname"
 require "fileutils"
+require "open3"
 
 module TestData
   class DumpsDatabase
@@ -21,7 +22,7 @@ module TestData
         database_name: @config.database_name,
         relative_path: @config.data_dump_path,
         full_path: @config.data_dump_full_path,
-        flags: @config.non_test_data_tables.map { |t| "-T #{t}" }.join(" ")
+        flags: (@config.non_test_data_tables + @config.dont_dump_these_tables).uniq.map { |t| "-T #{t} -T #{t}_id_seq" }.join(" ")
       )
 
       dump(
@@ -30,7 +31,7 @@ module TestData
         database_name: @config.database_name,
         relative_path: @config.non_test_data_dump_path,
         full_path: @config.non_test_data_dump_full_path,
-        flags: @config.non_test_data_tables.map { |t| "-t #{t}" }.join(" ")
+        flags: (@config.non_test_data_tables - @config.dont_dump_these_tables).uniq.map { |t| "-t #{t}" }.join(" ")
       )
     end
 
@@ -39,11 +40,60 @@ module TestData
     def dump(type:, database_name:, relative_path:, full_path:, name: type, flags: "")
       dump_pathname = Pathname.new(full_path)
       FileUtils.mkdir_p(File.dirname(dump_pathname))
-      if system "pg_dump #{database_name} --no-tablespaces --no-owner --inserts --#{type}-only #{flags} -f #{dump_pathname}"
-        puts "Dumped database '#{database_name}' #{name} to '#{relative_path}'"
+      before_size = File.size?(dump_pathname)
+      if execute("pg_dump #{database_name} --no-tablespaces --no-owner --inserts --#{type}-only #{flags} -f #{dump_pathname}")
+        prepend_set_replication_role!(full_path) if type == :data
+
+        TestData.log.info "Dumped '#{database_name}' #{name} to '#{relative_path}'"
+        log_size_info_and_warnings(before_size: before_size, after_size: File.size(dump_pathname))
+      else
+        raise Error.new("Failed while attempting to  dump '#{database_name}' #{name} to '#{relative_path}'")
+      end
+    end
+
+    def execute(command)
+      TestData.log.debug("Running SQL dump command:\n  #{command}")
+      stdout, stderr, status = Open3.capture3(command)
+      if status == 0
+        TestData.log.debug(stdout)
+        TestData.log.debug(stderr)
+        true
       else
-        raise "Failed while attempting to  dump '#{database_name}' #{name} to '#{relative_path}'"
+        TestData.log.info(stdout)
+        TestData.log.error(stderr)
+        false
       end
     end
+
+    def prepend_set_replication_role!(data_dump_path)
+      system <<~COMMAND
+        ed -s #{data_dump_path} <<EOF
+        1 s/^/set session_replication_role = replica;/
+        w
+        EOF
+      COMMAND
+      TestData.log.debug("Prepended replication role instruction to '#{data_dump_path}'")
+    end
+
+    def log_size_info_and_warnings(before_size:, after_size:)
+      percent_change = percent_change(before_size, after_size)
+      TestData.log.info "  Size: #{to_size(after_size)}#{" (#{percent_change}% #{before_size > after_size ? "decrease" : "increase"})" if percent_change}"
+      if after_size > 5242880
+        TestData.log.warn "  WARNING: file size exceeds 5MB. Be sure to only persist what data you need to sufficiently test your application"
+      end
+      if before_size && (after_size - before_size) > 1048576
+        TestData.log.warn "  WARNING: size of this dump increased by #{to_size(after_size - before_size)}. You may want to inspect the file to validate extraneous data was not committed"
+      end
+    end
+
+    def percent_change(before_size, after_size)
+      return unless before_size && before_size > 0 && after_size
+      ((before_size - after_size).abs / before_size * 100).round(2)
+    end
+
+    def to_size(bytes)
+      e = Math.log10(bytes).to_i / 3
+      "%.0f" % (bytes / 1000**e) + [" bytes", "KB", "MB", "GB"][e]
+    end
   end
 end