test_data 0.0.2 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +13 -0
  3. data/Gemfile.lock +15 -15
  4. data/LICENSE.txt +1 -6
  5. data/README.md +652 -276
  6. data/example/Gemfile.lock +73 -73
  7. data/example/test/integration/better_mode_switching_demo_test.rb +4 -0
  8. data/example/test/integration/parallel_boops_with_fixtures_test.rb +2 -2
  9. data/example/test/integration/parallel_boops_without_fixtures_test.rb +2 -2
  10. data/example/test/integration/rails_fixtures_double_load_test.rb +10 -0
  11. data/example/test/integration/rails_fixtures_override_test.rb +127 -0
  12. data/example/test/integration/transaction_committing_boops_test.rb +14 -3
  13. data/example/test/test_helper.rb +2 -2
  14. data/lib/generators/test_data/cable_yaml_generator.rb +18 -0
  15. data/lib/generators/test_data/database_yaml_generator.rb +2 -3
  16. data/lib/generators/test_data/environment_file_generator.rb +3 -0
  17. data/lib/generators/test_data/initializer_generator.rb +7 -0
  18. data/lib/generators/test_data/secrets_yaml_generator.rb +19 -0
  19. data/lib/generators/test_data/webpacker_yaml_generator.rb +3 -2
  20. data/lib/test_data.rb +5 -0
  21. data/lib/test_data/active_record_ext.rb +11 -0
  22. data/lib/test_data/config.rb +14 -2
  23. data/lib/test_data/configurators.rb +2 -0
  24. data/lib/test_data/configurators/cable_yaml.rb +25 -0
  25. data/lib/test_data/configurators/environment_file.rb +3 -2
  26. data/lib/test_data/configurators/initializer.rb +3 -2
  27. data/lib/test_data/configurators/secrets_yaml.rb +25 -0
  28. data/lib/test_data/configurators/webpacker_yaml.rb +4 -3
  29. data/lib/test_data/dumps_database.rb +24 -1
  30. data/lib/test_data/generator_support.rb +3 -0
  31. data/lib/test_data/loads_database_dumps.rb +1 -1
  32. data/lib/test_data/log.rb +19 -1
  33. data/lib/test_data/rake.rb +16 -6
  34. data/lib/test_data/statistics.rb +6 -1
  35. data/lib/test_data/transactional_data_loader.rb +156 -46
  36. data/lib/test_data/version.rb +1 -1
  37. data/script/test +11 -0
  38. data/test_data.gemspec +1 -1
  39. metadata +11 -3
@@ -8,11 +8,11 @@ class SerializedNonTransactionalTestCase < ActiveSupport::TestCase
8
8
  parallelize(workers: 1)
9
9
  self.use_transactional_tests = false
10
10
 
11
- def setup
11
+ setup do
12
12
  TestData.load
13
13
  end
14
14
 
15
- def teardown
15
+ teardown do
16
16
  TestData.rollback
17
17
  end
18
18
  end
@@ -0,0 +1,18 @@
1
+ require "rails/generators"
2
+ require_relative "../../test_data/generator_support"
3
+
4
+ module TestData
5
+ class CableYamlGenerator < Rails::Generators::Base
6
+ def call
7
+ unless Configurators::CableYaml.new.verify.looks_good?
8
+ inject_into_file "config/cable.yml", before: BEFORE_TEST_STANZA_REGEX do
9
+ <<~YAML
10
+
11
+ test_data:
12
+ adapter: async
13
+ YAML
14
+ end
15
+ end
16
+ end
17
+ end
18
+ end
@@ -1,15 +1,14 @@
1
1
  require "rails/generators"
2
+ require_relative "../../test_data/generator_support"
2
3
 
3
4
  module TestData
4
5
  class DatabaseYamlGenerator < Rails::Generators::Base
5
- BEFORE_TEST_DATABASE_STANZA_REGEX = /^$\n(?:^\#.*\n)*^test:/
6
-
7
6
  def call
8
7
  if Configurators::DatabaseYaml.new.verify.looks_good?
9
8
  TestData.log.info "'test_data' section already defined in config/database.yml"
10
9
  else
11
10
  app_name = Rails.application.railtie_name.chomp("_application")
12
- inject_into_file "config/database.yml", before: BEFORE_TEST_DATABASE_STANZA_REGEX do
11
+ inject_into_file "config/database.yml", before: BEFORE_TEST_STANZA_REGEX do
13
12
  <<~YAML
14
13
 
15
14
  # Used in conjunction with the test_data gem
@@ -1,12 +1,15 @@
1
1
  require "rails/generators"
2
+ require_relative "../../test_data/generator_support"
2
3
 
3
4
  module TestData
4
5
  class EnvironmentFileGenerator < Rails::Generators::Base
5
6
  def call
6
7
  create_file "config/environments/test_data.rb", <<~RUBY
8
+ # Load the development environment as a starting point
7
9
  require_relative "development"
8
10
 
9
11
  Rails.application.configure do
12
+ # Don't persist schema.rb or structure.sql after test_data is migrated
10
13
  config.active_record.dump_schema_after_migration = false
11
14
  end
12
15
  RUBY
@@ -1,4 +1,5 @@
1
1
  require "rails/generators"
2
+ require_relative "../../test_data/generator_support"
2
3
 
3
4
  module TestData
4
5
  class InitializerGenerator < Rails::Generators::Base
@@ -28,6 +29,12 @@ module TestData
28
29
  # `data_dump_path` will be truncated
29
30
  # config.truncate_these_test_data_tables = nil
30
31
 
32
+ # Perform TestData.load and TestData.truncate inside nested
33
+ # transactions for increased test isolation and speed. Setting this
34
+ # to false will disable several features that depend on transactions
35
+ # being used
36
+ # config.use_transactional_data_loader = true
37
+
31
38
  # Log level (valid values: [:debug, :info, :warn, :error, :quiet])
32
39
  # Can also be set with env var TEST_DATA_LOG_LEVEL
33
40
  # config.log_level = :info
@@ -0,0 +1,19 @@
1
+ require "rails/generators"
2
+ require_relative "../../test_data/generator_support"
3
+
4
+ module TestData
5
+ class SecretsYamlGenerator < Rails::Generators::Base
6
+ def call
7
+ unless Configurators::SecretsYaml.new.verify.looks_good?
8
+ inject_into_file "config/secrets.yml", before: BEFORE_TEST_STANZA_REGEX do
9
+ <<~YAML
10
+
11
+ # Simplify configuration with the test_data environment
12
+ test_data:
13
+ secret_key_base: #{SecureRandom.hex(64)}
14
+ YAML
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end
@@ -1,9 +1,9 @@
1
1
  require "rails/generators"
2
+ require_relative "../../test_data/generator_support"
2
3
 
3
4
  module TestData
4
5
  class WebpackerYamlGenerator < Rails::Generators::Base
5
6
  AFTER_DEVELOPMENT_WEBPACK_STANZA_REGEX = /^development:/
6
- BEFORE_TEST_WEBPACK_STANZA_REGEX = /^$\n(?:^\#.*\n)*^test:/
7
7
 
8
8
  def call
9
9
  if Configurators::WebpackerYaml.new.verify.looks_good?
@@ -12,12 +12,13 @@ module TestData
12
12
  inject_into_file "config/webpacker.yml", after: AFTER_DEVELOPMENT_WEBPACK_STANZA_REGEX do
13
13
  " &development"
14
14
  end
15
- inject_into_file "config/webpacker.yml", before: BEFORE_TEST_WEBPACK_STANZA_REGEX do
15
+ inject_into_file "config/webpacker.yml", before: BEFORE_TEST_STANZA_REGEX do
16
16
  <<~YAML
17
17
 
18
18
  # Used in conjunction with the test_data gem
19
19
  test_data:
20
20
  <<: *development
21
+
21
22
  YAML
22
23
  end
23
24
  end
data/lib/test_data.rb CHANGED
@@ -1,10 +1,13 @@
1
+ require_relative "test_data/active_record_ext"
1
2
  require_relative "test_data/active_support_ext"
2
3
  require_relative "test_data/config"
3
4
  require_relative "test_data/configuration_verification"
4
5
  require_relative "test_data/configurators"
5
6
  require_relative "test_data/configurators/environment_file"
6
7
  require_relative "test_data/configurators/initializer"
8
+ require_relative "test_data/configurators/cable_yaml"
7
9
  require_relative "test_data/configurators/database_yaml"
10
+ require_relative "test_data/configurators/secrets_yaml"
8
11
  require_relative "test_data/configurators/webpacker_yaml"
9
12
  require_relative "test_data/detects_database_emptiness"
10
13
  require_relative "test_data/dumps_database"
@@ -21,5 +24,7 @@ require_relative "test_data/verifies_dumps_are_loadable"
21
24
  require_relative "test_data/version"
22
25
  require_relative "generators/test_data/environment_file_generator"
23
26
  require_relative "generators/test_data/initializer_generator"
27
+ require_relative "generators/test_data/cable_yaml_generator"
24
28
  require_relative "generators/test_data/database_yaml_generator"
29
+ require_relative "generators/test_data/secrets_yaml_generator"
25
30
  require_relative "generators/test_data/webpacker_yaml_generator"
@@ -0,0 +1,11 @@
1
+ module TestData
2
+ def self.prevent_rails_fixtures_from_loading_automatically!
3
+ ActiveRecord::TestFixtures.define_method(:__test_data_gem_setup_fixtures,
4
+ ActiveRecord::TestFixtures.instance_method(:setup_fixtures))
5
+ ActiveRecord::TestFixtures.remove_method(:setup_fixtures)
6
+ ActiveRecord::TestFixtures.define_method(:setup_fixtures, ->(config = nil) {})
7
+
8
+ ActiveRecord::TestFixtures.remove_method(:teardown_fixtures)
9
+ ActiveRecord::TestFixtures.define_method(:teardown_fixtures, -> {})
10
+ end
11
+ end
@@ -32,6 +32,15 @@ module TestData
32
32
  # Tables to truncate when TestData.truncate is called
33
33
  attr_accessor :truncate_these_test_data_tables
34
34
 
35
+ # Perform TestData.load and TestData.truncate inside nested
36
+ # transactions for increased test isolation and speed. Setting this to false
37
+ # will disable several features that depend on transactions being used
38
+ attr_reader :use_transactional_data_loader
39
+ def use_transactional_data_loader=(use_transactions)
40
+ TestData.ensure_we_dont_mix_transactional_and_non_transactional_data_loaders!(use_transactions)
41
+ @use_transactional_data_loader = use_transactions
42
+ end
43
+
35
44
  # Log level (valid values: [:debug, :info, :warn, :error, :quiet])
36
45
  def log_level
37
46
  TestData.log.level
@@ -41,7 +50,7 @@ module TestData
41
50
  TestData.log.level = level
42
51
  end
43
52
 
44
- attr_reader :pwd, :database_yaml_path
53
+ attr_reader :pwd, :cable_yaml_path, :database_yaml_path, :secrets_yaml_path
45
54
 
46
55
  def self.full_path_reader(*relative_path_readers)
47
56
  relative_path_readers.each do |relative_path_reader|
@@ -51,17 +60,20 @@ module TestData
51
60
  end
52
61
  end
53
62
 
54
- full_path_reader :schema_dump_path, :data_dump_path, :non_test_data_dump_path, :database_yaml_path
63
+ full_path_reader :schema_dump_path, :data_dump_path, :non_test_data_dump_path, :cable_yaml_path, :database_yaml_path, :secrets_yaml_path
55
64
 
56
65
  def initialize(pwd:)
57
66
  @pwd = pwd
58
67
  @schema_dump_path = "test/support/test_data/schema.sql"
59
68
  @data_dump_path = "test/support/test_data/data.sql"
60
69
  @non_test_data_dump_path = "test/support/test_data/non_test_data.sql"
70
+ @cable_yaml_path = "config/cable.yml"
61
71
  @database_yaml_path = "config/database.yml"
72
+ @secrets_yaml_path = "config/secrets.yml"
62
73
  @non_test_data_tables = []
63
74
  @dont_dump_these_tables = []
64
75
  @truncate_these_test_data_tables = nil
76
+ @use_transactional_data_loader = true
65
77
  end
66
78
 
67
79
  def database_yaml
@@ -4,7 +4,9 @@ module TestData
4
4
  [
5
5
  EnvironmentFile,
6
6
  Initializer,
7
+ CableYaml,
7
8
  DatabaseYaml,
9
+ SecretsYaml,
8
10
  WebpackerYaml
9
11
  ].map(&:new)
10
12
  end
@@ -0,0 +1,25 @@
1
+ module TestData
2
+ module Configurators
3
+ class CableYaml
4
+ def initialize
5
+ @generator = CableYamlGenerator.new
6
+ @config = TestData.config
7
+ end
8
+
9
+ def verify
10
+ if !File.exist?(@config.cable_yaml_full_path) ||
11
+ YAML.load_file(@config.cable_yaml_full_path).key?("test_data")
12
+ ConfigurationVerification.new(looks_good?: true)
13
+ else
14
+ ConfigurationVerification.new(problems: [
15
+ "'#{@config.cable_yaml_path}' exists but does not contain a 'test_data' section"
16
+ ])
17
+ end
18
+ end
19
+
20
+ def configure
21
+ @generator.call
22
+ end
23
+ end
24
+ end
25
+ end
@@ -7,12 +7,13 @@ module TestData
7
7
  end
8
8
 
9
9
  def verify
10
- pathname = Pathname.new("#{@config.pwd}/config/environments/test_data.rb")
10
+ path = "config/environments/test_data.rb"
11
+ pathname = Pathname.new("#{@config.pwd}/#{path}")
11
12
  if pathname.readable?
12
13
  ConfigurationVerification.new(looks_good?: true)
13
14
  else
14
15
  ConfigurationVerification.new(problems: [
15
- "'#{pathname}' is not readable"
16
+ "'#{path}' is not readable"
16
17
  ])
17
18
  end
18
19
  end
@@ -7,12 +7,13 @@ module TestData
7
7
  end
8
8
 
9
9
  def verify
10
- pathname = Pathname.new("#{@config.pwd}/config/initializers/test_data.rb")
10
+ path = "config/initializers/test_data.rb"
11
+ pathname = Pathname.new("#{@config.pwd}/#{path}")
11
12
  if pathname.readable?
12
13
  ConfigurationVerification.new(looks_good?: true)
13
14
  else
14
15
  ConfigurationVerification.new(problems: [
15
- "'#{pathname}' is not readable"
16
+ "'#{path}' is not readable"
16
17
  ])
17
18
  end
18
19
  end
@@ -0,0 +1,25 @@
1
+ module TestData
2
+ module Configurators
3
+ class SecretsYaml
4
+ def initialize
5
+ @generator = SecretsYamlGenerator.new
6
+ @config = TestData.config
7
+ end
8
+
9
+ def verify
10
+ if !File.exist?(@config.secrets_yaml_full_path) ||
11
+ YAML.load_file(@config.secrets_yaml_full_path).key?("test_data")
12
+ ConfigurationVerification.new(looks_good?: true)
13
+ else
14
+ ConfigurationVerification.new(problems: [
15
+ "'#{@config.secrets_yaml_path}' exists but does not contain a 'test_data' section"
16
+ ])
17
+ end
18
+ end
19
+
20
+ def configure
21
+ @generator.call
22
+ end
23
+ end
24
+ end
25
+ end
@@ -7,16 +7,17 @@ module TestData
7
7
  end
8
8
 
9
9
  def verify
10
- pathname = Pathname.new("#{@config.pwd}/config/webpacker.yml")
10
+ path = "config/webpacker.yml"
11
+ pathname = Pathname.new("#{@config.pwd}/#{path}")
11
12
  return ConfigurationVerification.new(looks_good?: true) unless pathname.readable?
12
13
  yaml = load_yaml(pathname)
13
14
  if yaml.nil?
14
15
  ConfigurationVerification.new(problems: [
15
- "'#{pathname}' is not valid YAML"
16
+ "'#{path}' is not valid YAML"
16
17
  ])
17
18
  elsif !yaml.key?("test_data")
18
19
  ConfigurationVerification.new(problems: [
19
- "'#{pathname}' does not contain a 'test_data' section"
20
+ "'#{path}' does not contain a 'test_data' section"
20
21
  ])
21
22
  else
22
23
  ConfigurationVerification.new(looks_good?: true)
@@ -40,12 +40,14 @@ module TestData
40
40
  def dump(type:, database_name:, relative_path:, full_path:, name: type, flags: "")
41
41
  dump_pathname = Pathname.new(full_path)
42
42
  FileUtils.mkdir_p(File.dirname(dump_pathname))
43
+ before_size = File.size?(dump_pathname)
43
44
  if execute("pg_dump #{database_name} --no-tablespaces --no-owner --inserts --#{type}-only #{flags} -f #{dump_pathname}")
44
45
  prepend_set_replication_role!(full_path) if type == :data
45
46
 
46
47
  TestData.log.info "Dumped '#{database_name}' #{name} to '#{relative_path}'"
48
+ log_size_info_and_warnings(before_size: before_size, after_size: File.size(dump_pathname))
47
49
  else
48
- raise "Failed while attempting to dump '#{database_name}' #{name} to '#{relative_path}'"
50
+ raise Error.new("Failed while attempting to dump '#{database_name}' #{name} to '#{relative_path}'")
49
51
  end
50
52
  end
51
53
 
@@ -72,5 +74,26 @@ module TestData
72
74
  COMMAND
73
75
  TestData.log.debug("Prepended replication role instruction to '#{data_dump_path}'")
74
76
  end
77
+
78
+ def log_size_info_and_warnings(before_size:, after_size:)
79
+ percent_change = percent_change(before_size, after_size)
80
+ TestData.log.info " Size: #{to_size(after_size)}#{" (#{percent_change}% #{before_size > after_size ? "decrease" : "increase"})" if percent_change}"
81
+ if after_size > 5242880
82
+ TestData.log.warn " WARNING: file size exceeds 5MB. Be sure to only persist what data you need to sufficiently test your application"
83
+ end
84
+ if before_size && (after_size - before_size) > 1048576
85
+ TestData.log.warn " WARNING: size of this dump increased by #{to_size(after_size - before_size)}. You may want to inspect the file to validate extraneous data was not committed"
86
+ end
87
+ end
88
+
89
+ def percent_change(before_size, after_size)
90
+ return unless before_size && before_size > 0 && after_size
91
+ ((before_size - after_size).abs / before_size * 100).round(2)
92
+ end
93
+
94
+ def to_size(bytes)
95
+ e = Math.log10(bytes).to_i / 3
96
+ "%.0f" % (bytes / 1000**e) + [" bytes", "KB", "MB", "GB"][e]
97
+ end
75
98
  end
76
99
  end
@@ -0,0 +1,3 @@
1
+ module TestData
2
+ BEFORE_TEST_STANZA_REGEX = /^$\n(?:^\#.*\n)*^test:/
3
+ end
@@ -38,7 +38,7 @@ module TestData
38
38
  if system "psql -q -d #{database_name} < #{dump_pathname}"
39
39
  TestData.log.info "Loaded #{name} from '#{relative_path}' into database '#{database_name}' "
40
40
  else
41
- raise "Failed while attempting to load #{name} from '#{relative_path}' into database '#{database_name}'"
41
+ raise Error.new("Failed while attempting to load #{name} from '#{relative_path}' into database '#{database_name}'")
42
42
  end
43
43
  end
44
44
  end
data/lib/test_data/log.rb CHANGED
@@ -6,13 +6,20 @@ module TestData
6
6
  class Log
7
7
  LEVELS = [:debug, :info, :warn, :error, :quiet]
8
8
  DEFAULT_WRITER = ->(message, level) do
9
- output = "[test_data: #{level}] #{message}"
9
+ output = "[test_data:#{level}] #{message}"
10
10
  if [:warn, :error].include?(level)
11
11
  warn output
12
12
  else
13
13
  puts output
14
14
  end
15
15
  end
16
+ PLAIN_WRITER = ->(message, level) do
17
+ if [:warn, :error].include?(level)
18
+ warn message
19
+ else
20
+ puts message
21
+ end
22
+ end
16
23
 
17
24
  attr_reader :level, :writer
18
25
 
@@ -49,6 +56,17 @@ module TestData
49
56
  end
50
57
  end
51
58
 
59
+ def with_writer(writer, &blk)
60
+ og_writer = self.writer
61
+ self.writer = writer
62
+ blk.call
63
+ self.writer = og_writer
64
+ end
65
+
66
+ def with_plain_writer(&blk)
67
+ with_writer(PLAIN_WRITER, &blk)
68
+ end
69
+
52
70
  private
53
71
 
54
72
  def enabled?(level)
@@ -22,13 +22,15 @@ end
22
22
 
23
23
  desc "Verifies test_data environment looks good"
24
24
  task "test_data:verify_config" do
25
- config = TestData::VerifiesConfiguration.new.call
26
- unless config.looks_good?
27
- TestData.log.warn "The test_data gem is not configured correctly. Try 'rake test_data:configure'?\n"
28
- config.problems.each do |problem|
29
- TestData.log.warn " - #{problem}"
25
+ TestData.log.with_plain_writer do
26
+ config = TestData::VerifiesConfiguration.new.call
27
+ unless config.looks_good?
28
+ TestData.log.warn "\nThe test_data gem is not configured correctly. Try running: rake test_data:configure\n\n"
29
+ config.problems.each do |problem|
30
+ TestData.log.warn " - #{problem}"
31
+ end
32
+ fail
30
33
  end
31
- fail
32
34
  end
33
35
  end
34
36
 
@@ -48,6 +50,14 @@ task "test_data:initialize" => ["test_data:verify_config", :environment] do
48
50
  ActiveRecord::Tasks::DatabaseTasks.load_schema_current(ActiveRecord::Base.schema_format, ENV["SCHEMA"], "test_data")
49
51
  ActiveRecord::Tasks::DatabaseTasks.load_seed
50
52
  end
53
+
54
+ TestData.log.info <<~MSG
55
+ Your test_data environment and database are ready for use! You can now run
56
+ your server (or any command) to create some test data like so:
57
+
58
+ $ RAILS_ENV=test_data bin/rails server
59
+
60
+ MSG
51
61
  end
52
62
 
53
63
  desc "Initialize test_data Rails environment & configure database"