test_data 0.0.2 → 0.1.0

data/example/test/test_helper.rb

@@ -8,11 +8,11 @@ class SerializedNonTransactionalTestCase < ActiveSupport::TestCase
   parallelize(workers: 1)
   self.use_transactional_tests = false
 
-  def setup
+  setup do
     TestData.load
   end
 
-  def teardown
+  teardown do
     TestData.rollback
   end
 end

data/lib/generators/test_data/cable_yaml_generator.rb

@@ -0,0 +1,18 @@
+require "rails/generators"
+require_relative "../../test_data/generator_support"
+
+module TestData
+  class CableYamlGenerator < Rails::Generators::Base
+    def call
+      unless Configurators::CableYaml.new.verify.looks_good?
+        inject_into_file "config/cable.yml", before: BEFORE_TEST_STANZA_REGEX do
+          <<~YAML
+
+            test_data:
+              adapter: async
+          YAML
+        end
+      end
+    end
+  end
+end

data/lib/generators/test_data/database_yaml_generator.rb

@@ -1,15 +1,14 @@
 require "rails/generators"
+require_relative "../../test_data/generator_support"
 
 module TestData
   class DatabaseYamlGenerator < Rails::Generators::Base
-    BEFORE_TEST_DATABASE_STANZA_REGEX = /^$\n(?:^\#.*\n)*^test:/
-
     def call
       if Configurators::DatabaseYaml.new.verify.looks_good?
         TestData.log.info "'test_data' section already defined in config/database.yml"
       else
         app_name = Rails.application.railtie_name.chomp("_application")
-        inject_into_file "config/database.yml", before: BEFORE_TEST_DATABASE_STANZA_REGEX do
+        inject_into_file "config/database.yml", before: BEFORE_TEST_STANZA_REGEX do
           <<~YAML
 
             # Used in conjunction with the test_data gem

data/lib/generators/test_data/environment_file_generator.rb

@@ -1,12 +1,15 @@
 require "rails/generators"
+require_relative "../../test_data/generator_support"
 
 module TestData
   class EnvironmentFileGenerator < Rails::Generators::Base
     def call
       create_file "config/environments/test_data.rb", <<~RUBY
+        # Load the development environment as a starting point
         require_relative "development"
 
         Rails.application.configure do
+          # Don't persist schema.rb or structure.sql after test_data is migrated
           config.active_record.dump_schema_after_migration = false
         end
       RUBY

data/lib/generators/test_data/initializer_generator.rb

@@ -1,4 +1,5 @@
 require "rails/generators"
+require_relative "../../test_data/generator_support"
 
 module TestData
   class InitializerGenerator < Rails::Generators::Base
@@ -28,6 +29,12 @@ module TestData
           #   `data_dump_path` will be truncated
           # config.truncate_these_test_data_tables = nil
 
+          # Perform TestData.load and TestData.truncate inside nested
+          #   transactions for increased test isolation and speed. Setting this
+          #   to false will disable several features that depend on transactions
+          #   being used
+          # config.use_transactional_data_loader = true
+
           # Log level (valid values: [:debug, :info, :warn, :error, :quiet])
           #   Can also be set with env var TEST_DATA_LOG_LEVEL
           # config.log_level = :info

data/lib/generators/test_data/secrets_yaml_generator.rb

@@ -0,0 +1,19 @@
+require "rails/generators"
+require_relative "../../test_data/generator_support"
+
+module TestData
+  class SecretsYamlGenerator < Rails::Generators::Base
+    def call
+      unless Configurators::SecretsYaml.new.verify.looks_good?
+        inject_into_file "config/secrets.yml", before: BEFORE_TEST_STANZA_REGEX do
+          <<~YAML
+
+            # Simplify configuration with the test_data environment
+            test_data:
+              secret_key_base: #{SecureRandom.hex(64)}
+          YAML
+        end
+      end
+    end
+  end
+end

data/lib/generators/test_data/webpacker_yaml_generator.rb

@@ -1,9 +1,9 @@
 require "rails/generators"
+require_relative "../../test_data/generator_support"
 
 module TestData
   class WebpackerYamlGenerator < Rails::Generators::Base
     AFTER_DEVELOPMENT_WEBPACK_STANZA_REGEX = /^development:/
-    BEFORE_TEST_WEBPACK_STANZA_REGEX = /^$\n(?:^\#.*\n)*^test:/
 
     def call
       if Configurators::WebpackerYaml.new.verify.looks_good?
@@ -12,12 +12,13 @@ module TestData
         inject_into_file "config/webpacker.yml", after: AFTER_DEVELOPMENT_WEBPACK_STANZA_REGEX do
           " &development"
         end
-        inject_into_file "config/webpacker.yml", before: BEFORE_TEST_WEBPACK_STANZA_REGEX do
+        inject_into_file "config/webpacker.yml", before: BEFORE_TEST_STANZA_REGEX do
           <<~YAML
 
             # Used in conjunction with the test_data gem
             test_data:
               <<: *development
+
           YAML
         end
       end

data/lib/test_data.rb

@@ -1,10 +1,13 @@
+require_relative "test_data/active_record_ext"
 require_relative "test_data/active_support_ext"
 require_relative "test_data/config"
 require_relative "test_data/configuration_verification"
 require_relative "test_data/configurators"
 require_relative "test_data/configurators/environment_file"
 require_relative "test_data/configurators/initializer"
+require_relative "test_data/configurators/cable_yaml"
 require_relative "test_data/configurators/database_yaml"
+require_relative "test_data/configurators/secrets_yaml"
 require_relative "test_data/configurators/webpacker_yaml"
 require_relative "test_data/detects_database_emptiness"
 require_relative "test_data/dumps_database"
@@ -21,5 +24,7 @@ require_relative "test_data/verifies_dumps_are_loadable"
 require_relative "test_data/version"
 require_relative "generators/test_data/environment_file_generator"
 require_relative "generators/test_data/initializer_generator"
+require_relative "generators/test_data/cable_yaml_generator"
 require_relative "generators/test_data/database_yaml_generator"
+require_relative "generators/test_data/secrets_yaml_generator"
 require_relative "generators/test_data/webpacker_yaml_generator"

data/lib/test_data/active_record_ext.rb

@@ -0,0 +1,11 @@
+module TestData
+  def self.prevent_rails_fixtures_from_loading_automatically!
+    ActiveRecord::TestFixtures.define_method(:__test_data_gem_setup_fixtures,
+      ActiveRecord::TestFixtures.instance_method(:setup_fixtures))
+    ActiveRecord::TestFixtures.remove_method(:setup_fixtures)
+    ActiveRecord::TestFixtures.define_method(:setup_fixtures, ->(config = nil) {})
+
+    ActiveRecord::TestFixtures.remove_method(:teardown_fixtures)
+    ActiveRecord::TestFixtures.define_method(:teardown_fixtures, -> {})
+  end
+end

data/lib/test_data/config.rb

@@ -32,6 +32,15 @@ module TestData
     # Tables to truncate when TestData.truncate is called
     attr_accessor :truncate_these_test_data_tables
 
+    # Perform TestData.load and TestData.truncate inside nested
+    # transactions for increased test isolation and speed. Setting this to false
+    # will disable several features that depend on transactions being used
+    attr_reader :use_transactional_data_loader
+    def use_transactional_data_loader=(use_transactions)
+      TestData.ensure_we_dont_mix_transactional_and_non_transactional_data_loaders!(use_transactions)
+      @use_transactional_data_loader = use_transactions
+    end
+
     # Log level (valid values: [:debug, :info, :warn, :error, :quiet])
     def log_level
       TestData.log.level
@@ -41,7 +50,7 @@ module TestData
       TestData.log.level = level
     end
 
-    attr_reader :pwd, :database_yaml_path
+    attr_reader :pwd, :cable_yaml_path, :database_yaml_path, :secrets_yaml_path
 
     def self.full_path_reader(*relative_path_readers)
       relative_path_readers.each do |relative_path_reader|
@@ -51,17 +60,20 @@ module TestData
       end
     end
 
-    full_path_reader :schema_dump_path, :data_dump_path, :non_test_data_dump_path, :database_yaml_path
+    full_path_reader :schema_dump_path, :data_dump_path, :non_test_data_dump_path, :cable_yaml_path, :database_yaml_path, :secrets_yaml_path
 
     def initialize(pwd:)
       @pwd = pwd
       @schema_dump_path = "test/support/test_data/schema.sql"
       @data_dump_path = "test/support/test_data/data.sql"
       @non_test_data_dump_path = "test/support/test_data/non_test_data.sql"
+      @cable_yaml_path = "config/cable.yml"
       @database_yaml_path = "config/database.yml"
+      @secrets_yaml_path = "config/secrets.yml"
       @non_test_data_tables = []
       @dont_dump_these_tables = []
       @truncate_these_test_data_tables = nil
+      @use_transactional_data_loader = true
     end
 
     def database_yaml

data/lib/test_data/configurators.rb

@@ -4,7 +4,9 @@ module TestData
       [
         EnvironmentFile,
         Initializer,
+        CableYaml,
         DatabaseYaml,
+        SecretsYaml,
         WebpackerYaml
       ].map(&:new)
     end

data/lib/test_data/configurators/cable_yaml.rb

@@ -0,0 +1,25 @@
+module TestData
+  module Configurators
+    class CableYaml
+      def initialize
+        @generator = CableYamlGenerator.new
+        @config = TestData.config
+      end
+
+      def verify
+        if !File.exist?(@config.cable_yaml_full_path) ||
+            YAML.load_file(@config.cable_yaml_full_path).key?("test_data")
+          ConfigurationVerification.new(looks_good?: true)
+        else
+          ConfigurationVerification.new(problems: [
+            "'#{@config.cable_yaml_path}' exists but does not contain a 'test_data' section"
+          ])
+        end
+      end
+
+      def configure
+        @generator.call
+      end
+    end
+  end
+end

data/lib/test_data/configurators/environment_file.rb

@@ -7,12 +7,13 @@ module TestData
       end
 
       def verify
-        pathname = Pathname.new("#{@config.pwd}/config/environments/test_data.rb")
+        path = "config/environments/test_data.rb"
+        pathname = Pathname.new("#{@config.pwd}/#{path}")
         if pathname.readable?
           ConfigurationVerification.new(looks_good?: true)
         else
           ConfigurationVerification.new(problems: [
-            "'#{pathname}' is not readable"
+            "'#{path}' is not readable"
           ])
         end
       end

data/lib/test_data/configurators/initializer.rb

@@ -7,12 +7,13 @@ module TestData
       end
 
       def verify
-        pathname = Pathname.new("#{@config.pwd}/config/initializers/test_data.rb")
+        path = "config/initializers/test_data.rb"
+        pathname = Pathname.new("#{@config.pwd}/#{path}")
         if pathname.readable?
           ConfigurationVerification.new(looks_good?: true)
         else
           ConfigurationVerification.new(problems: [
-            "'#{pathname}' is not readable"
+            "'#{path}' is not readable"
           ])
         end
       end

data/lib/test_data/configurators/secrets_yaml.rb

@@ -0,0 +1,25 @@
+module TestData
+  module Configurators
+    class SecretsYaml
+      def initialize
+        @generator = SecretsYamlGenerator.new
+        @config = TestData.config
+      end
+
+      def verify
+        if !File.exist?(@config.secrets_yaml_full_path) ||
+            YAML.load_file(@config.secrets_yaml_full_path).key?("test_data")
+          ConfigurationVerification.new(looks_good?: true)
+        else
+          ConfigurationVerification.new(problems: [
+            "'#{@config.secrets_yaml_path}' exists but does not contain a 'test_data' section"
+          ])
+        end
+      end
+
+      def configure
+        @generator.call
+      end
+    end
+  end
+end

data/lib/test_data/configurators/webpacker_yaml.rb

@@ -7,16 +7,17 @@ module TestData
       end
 
       def verify
-        pathname = Pathname.new("#{@config.pwd}/config/webpacker.yml")
+        path = "config/webpacker.yml"
+        pathname = Pathname.new("#{@config.pwd}/#{path}")
         return ConfigurationVerification.new(looks_good?: true) unless pathname.readable?
         yaml = load_yaml(pathname)
         if yaml.nil?
           ConfigurationVerification.new(problems: [
-            "'#{pathname}' is not valid YAML"
+            "'#{path}' is not valid YAML"
           ])
         elsif !yaml.key?("test_data")
           ConfigurationVerification.new(problems: [
-            "'#{pathname}' does not contain a 'test_data' section"
+            "'#{path}' does not contain a 'test_data' section"
           ])
         else
           ConfigurationVerification.new(looks_good?: true)

data/lib/test_data/dumps_database.rb

@@ -40,12 +40,14 @@ module TestData
     def dump(type:, database_name:, relative_path:, full_path:, name: type, flags: "")
       dump_pathname = Pathname.new(full_path)
       FileUtils.mkdir_p(File.dirname(dump_pathname))
+      before_size = File.size?(dump_pathname)
       if execute("pg_dump #{database_name} --no-tablespaces --no-owner --inserts --#{type}-only #{flags} -f #{dump_pathname}")
         prepend_set_replication_role!(full_path) if type == :data
 
         TestData.log.info "Dumped '#{database_name}' #{name} to '#{relative_path}'"
+        log_size_info_and_warnings(before_size: before_size, after_size: File.size(dump_pathname))
       else
-        raise "Failed while attempting to  dump '#{database_name}' #{name} to '#{relative_path}'"
+        raise Error.new("Failed while attempting to  dump '#{database_name}' #{name} to '#{relative_path}'")
       end
     end
 
@@ -72,5 +74,26 @@ module TestData
       COMMAND
       TestData.log.debug("Prepended replication role instruction to '#{data_dump_path}'")
     end
+
+    def log_size_info_and_warnings(before_size:, after_size:)
+      percent_change = percent_change(before_size, after_size)
+      TestData.log.info "  Size: #{to_size(after_size)}#{" (#{percent_change}% #{before_size > after_size ? "decrease" : "increase"})" if percent_change}"
+      if after_size > 5242880
+        TestData.log.warn "  WARNING: file size exceeds 5MB. Be sure to only persist what data you need to sufficiently test your application"
+      end
+      if before_size && (after_size - before_size) > 1048576
+        TestData.log.warn "  WARNING: size of this dump increased by #{to_size(after_size - before_size)}. You may want to inspect the file to validate extraneous data was not committed"
+      end
+    end
+
+    def percent_change(before_size, after_size)
+      return unless before_size && before_size > 0 && after_size
+      ((before_size - after_size).abs / before_size * 100).round(2)
+    end
+
+    def to_size(bytes)
+      e = Math.log10(bytes).to_i / 3
+      "%.0f" % (bytes / 1000**e) + [" bytes", "KB", "MB", "GB"][e]
+    end
   end
 end

data/lib/test_data/generator_support.rb

@@ -0,0 +1,3 @@
+module TestData
+  BEFORE_TEST_STANZA_REGEX = /^$\n(?:^\#.*\n)*^test:/
+end

data/lib/test_data/loads_database_dumps.rb

@@ -38,7 +38,7 @@ module TestData
       if system "psql -q -d #{database_name} < #{dump_pathname}"
         TestData.log.info "Loaded #{name} from '#{relative_path}' into database '#{database_name}' "
       else
-        raise "Failed while attempting to load #{name} from '#{relative_path}' into database '#{database_name}'"
+        raise Error.new("Failed while attempting to load #{name} from '#{relative_path}' into database '#{database_name}'")
       end
     end
   end

data/lib/test_data/log.rb

@@ -6,13 +6,20 @@ module TestData
   class Log
     LEVELS = [:debug, :info, :warn, :error, :quiet]
     DEFAULT_WRITER = ->(message, level) do
-      output = "[test_data: #{level}] #{message}"
+      output = "[test_data:#{level}] #{message}"
       if [:warn, :error].include?(level)
         warn output
       else
         puts output
       end
     end
+    PLAIN_WRITER = ->(message, level) do
+      if [:warn, :error].include?(level)
+        warn message
+      else
+        puts message
+      end
+    end
 
     attr_reader :level, :writer
 
@@ -49,6 +56,17 @@ module TestData
       end
     end
 
+    def with_writer(writer, &blk)
+      og_writer = self.writer
+      self.writer = writer
+      blk.call
+      self.writer = og_writer
+    end
+
+    def with_plain_writer(&blk)
+      with_writer(PLAIN_WRITER, &blk)
+    end
+
     private
 
     def enabled?(level)

data/lib/test_data/rake.rb

@@ -22,13 +22,15 @@ end
 
 desc "Verifies test_data environment looks good"
 task "test_data:verify_config" do
-  config = TestData::VerifiesConfiguration.new.call
-  unless config.looks_good?
-    TestData.log.warn "The test_data gem is not configured correctly. Try 'rake test_data:configure'?\n"
-    config.problems.each do |problem|
-      TestData.log.warn "  - #{problem}"
+  TestData.log.with_plain_writer do
+    config = TestData::VerifiesConfiguration.new.call
+    unless config.looks_good?
+      TestData.log.warn "\nThe test_data gem is not configured correctly. Try running: rake test_data:configure\n\n"
+      config.problems.each do |problem|
+        TestData.log.warn "  - #{problem}"
+      end
+      fail
     end
-    fail
   end
 end
 
@@ -48,6 +50,14 @@ task "test_data:initialize" => ["test_data:verify_config", :environment] do
     ActiveRecord::Tasks::DatabaseTasks.load_schema_current(ActiveRecord::Base.schema_format, ENV["SCHEMA"], "test_data")
     ActiveRecord::Tasks::DatabaseTasks.load_seed
   end
+
+  TestData.log.info <<~MSG
+    Your test_data environment and database are ready for use! You can now run
+    your server (or any command) to create some test data like so:
+
+      $ RAILS_ENV=test_data bin/rails server
+
+  MSG
 end
 
 desc "Initialize test_data Rails environment & configure database"