active_record_data_loader 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +13 -0
  3. data/.rspec +3 -0
  4. data/.rubocop.yml +38 -0
  5. data/.travis.yml +11 -0
  6. data/Appraisals +17 -0
  7. data/CODE_OF_CONDUCT.md +74 -0
  8. data/Gemfile +8 -0
  9. data/Gemfile.lock +107 -0
  10. data/LICENSE.txt +21 -0
  11. data/README.md +49 -0
  12. data/Rakefile +10 -0
  13. data/active_record_data_loader.gemspec +47 -0
  14. data/bin/console +15 -0
  15. data/bin/setup +8 -0
  16. data/config/database.yml +10 -0
  17. data/config/database.yml.travis +7 -0
  18. data/gemfiles/.bundle/config +2 -0
  19. data/gemfiles/activerecord_5.gemfile +7 -0
  20. data/gemfiles/activerecord_6.gemfile +7 -0
  21. data/gemfiles/faker.gemfile +7 -0
  22. data/gemfiles/ffaker.gemfile +7 -0
  23. data/lib/active_record_data_loader.rb +64 -0
  24. data/lib/active_record_data_loader/active_record/belongs_to_configuration.rb +27 -0
  25. data/lib/active_record_data_loader/active_record/column_configuration.rb +43 -0
  26. data/lib/active_record_data_loader/active_record/enum_value_generator.rb +25 -0
  27. data/lib/active_record_data_loader/active_record/integer_value_generator.rb +21 -0
  28. data/lib/active_record_data_loader/active_record/model_data_generator.rb +72 -0
  29. data/lib/active_record_data_loader/active_record/polymorphic_belongs_to_configuration.rb +49 -0
  30. data/lib/active_record_data_loader/active_record/text_value_generator.rb +72 -0
  31. data/lib/active_record_data_loader/bulk_insert_strategy.rb +57 -0
  32. data/lib/active_record_data_loader/configuration.rb +17 -0
  33. data/lib/active_record_data_loader/copy_strategy.rb +58 -0
  34. data/lib/active_record_data_loader/data_faker.rb +85 -0
  35. data/lib/active_record_data_loader/dsl/definition.rb +25 -0
  36. data/lib/active_record_data_loader/dsl/model.rb +35 -0
  37. data/lib/active_record_data_loader/dsl/polymorphic_association.rb +25 -0
  38. data/lib/active_record_data_loader/loader.rb +66 -0
  39. data/lib/active_record_data_loader/version.rb +5 -0
  40. data/script/ci_build.sh +6 -0
  41. metadata +240 -0
@@ -0,0 +1,10 @@
1
+ postgres:
2
+ adapter: "postgresql"
3
+ host: "127.0.0.1"
4
+ database: "test"
5
+ username: "test"
6
+ password: "test"
7
+
8
+ sqlite3:
9
+ adapter: "sqlite3"
10
+ database: ":memory:"
@@ -0,0 +1,7 @@
1
+ postgres:
2
+ adapter: "postgresql"
3
+ database: "test"
4
+
5
+ sqlite3:
6
+ adapter: "sqlite3"
7
+ database: ":memory:"
@@ -0,0 +1,2 @@
1
+ ---
2
+ BUNDLE_RETRY: "1"
@@ -0,0 +1,7 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "activerecord", "~> 5.0"
6
+
7
+ gemspec path: "../"
@@ -0,0 +1,7 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "activerecord", "6.0.0.rc1"
6
+
7
+ gemspec path: "../"
@@ -0,0 +1,7 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "faker", ">= 1.9.3"
6
+
7
+ gemspec path: "../"
@@ -0,0 +1,7 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "ffaker"
6
+
7
+ gemspec path: "../"
@@ -0,0 +1,64 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "active_record_data_loader/version"
4
+ require "active_record"
5
+ require "active_record_data_loader/configuration"
6
+ require "active_record_data_loader/data_faker"
7
+ require "active_record_data_loader/active_record/integer_value_generator"
8
+ require "active_record_data_loader/active_record/text_value_generator"
9
+ require "active_record_data_loader/active_record/enum_value_generator"
10
+ require "active_record_data_loader/active_record/column_configuration"
11
+ require "active_record_data_loader/active_record/belongs_to_configuration"
12
+ require "active_record_data_loader/active_record/polymorphic_belongs_to_configuration"
13
+ require "active_record_data_loader/active_record/model_data_generator"
14
+ require "active_record_data_loader/dsl/polymorphic_association"
15
+ require "active_record_data_loader/dsl/model"
16
+ require "active_record_data_loader/dsl/definition"
17
+ require "active_record_data_loader/copy_strategy"
18
+ require "active_record_data_loader/bulk_insert_strategy"
19
+ require "active_record_data_loader/loader"
20
+
21
+ module ActiveRecordDataLoader
22
+ def self.define(config = ActiveRecordDataLoader.configuration, &block)
23
+ LoaderProxy.new(
24
+ configuration,
25
+ ActiveRecordDataLoader::Dsl::Definition.new(config).tap { |l| l.instance_eval(&block) }
26
+ )
27
+ end
28
+
29
+ def self.configure(&block)
30
+ @configuration = ActiveRecordDataLoader::Configuration.new.tap { |c| block.call(c) }
31
+ end
32
+
33
+ def self.configuration
34
+ @configuration ||= ActiveRecordDataLoader::Configuration.new
35
+ end
36
+
37
+ class LoaderProxy
38
+ def initialize(configuration, definition)
39
+ @configuration = configuration
40
+ @definition = definition
41
+ end
42
+
43
+ def load_data
44
+ definition.models.map do |m|
45
+ generator = ActiveRecordDataLoader::ActiveRecord::ModelDataGenerator.new(
46
+ model: m.klass,
47
+ column_settings: m.columns,
48
+ polymorphic_settings: m.polymorphic_associations
49
+ )
50
+
51
+ ActiveRecordDataLoader::Loader.load_data(
52
+ data_generator: generator,
53
+ batch_size: m.batch_size,
54
+ total_rows: m.row_count,
55
+ logger: configuration.logger
56
+ )
57
+ end
58
+ end
59
+
60
+ private
61
+
62
+ attr_reader :definition, :configuration
63
+ end
64
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveRecordDataLoader
4
+ module ActiveRecord
5
+ class BelongsToConfiguration
6
+ def self.config_for(ar_association:)
7
+ raise "#{name} does not support polymorphic associations" if ar_association.polymorphic?
8
+
9
+ { ar_association.join_foreign_key.to_sym => new(ar_association).foreign_key_func }
10
+ end
11
+
12
+ def initialize(ar_association)
13
+ @ar_association = ar_association
14
+ end
15
+
16
+ def foreign_key_func
17
+ -> { possible_values.sample }
18
+ end
19
+
20
+ private
21
+
22
+ def possible_values
23
+ @possible_values ||= @ar_association.klass.all.pluck(@ar_association.join_primary_key).to_a
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveRecordDataLoader
4
+ module ActiveRecord
5
+ class ColumnConfiguration
6
+ class << self
7
+ VALUE_GENERATORS = {
8
+ enum: EnumValueGenerator,
9
+ integer: IntegerValueGenerator,
10
+ string: TextValueGenerator,
11
+ text: TextValueGenerator,
12
+ }.freeze
13
+
14
+ def config_for(model_class:, ar_column:)
15
+ raise_error_if_not_supported(model_class, ar_column)
16
+
17
+ {
18
+ ar_column.name.to_sym => VALUE_GENERATORS[ar_column.type].generator_for(
19
+ model_class: model_class,
20
+ ar_column: ar_column
21
+ ),
22
+ }
23
+ end
24
+
25
+ def supported?(model_class:, ar_column:)
26
+ return false if model_class.reflect_on_association(ar_column.name)
27
+
28
+ VALUE_GENERATORS.keys.include?(ar_column.type)
29
+ end
30
+
31
+ private
32
+
33
+ def raise_error_if_not_supported(model_class, ar_column)
34
+ return if supported?(model_class: model_class, ar_column: ar_column)
35
+
36
+ raise <<~ERROR
37
+ Column '#{ar_column.name}' of type '#{ar_column.type}' in model '#{model_class.name}' not supported"
38
+ ERROR
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveRecordDataLoader
4
+ module ActiveRecord
5
+ class EnumValueGenerator
6
+ class << self
7
+ def generator_for(model_class:, ar_column:)
8
+ values = enum_values_for(model_class, ar_column.sql_type)
9
+ -> { values.sample }
10
+ end
11
+
12
+ private
13
+
14
+ def enum_values_for(model_class, enum_type)
15
+ model_class
16
+ .connection
17
+ .execute("SELECT unnest(enum_range(NULL::#{enum_type}))::text")
18
+ .map(&:values)
19
+ .flatten
20
+ .compact
21
+ end
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveRecordDataLoader
4
+ module ActiveRecord
5
+ class IntegerValueGenerator
6
+ class << self
7
+ def generator_for(model_class:, ar_column:)
8
+ range_limit = [(256**number_of_bytes(ar_column)) / 2 - 1, 1_000_000_000].min
9
+
10
+ -> { rand(0..range_limit) }
11
+ end
12
+
13
+ private
14
+
15
+ def number_of_bytes(ar_column)
16
+ ar_column.limit || 8
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,72 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveRecordDataLoader
4
+ module ActiveRecord
5
+ class ModelDataGenerator
6
+ attr_reader :table
7
+
8
+ def initialize(model:, column_settings:, polymorphic_settings: [])
9
+ @model_class = model
10
+ @table = model.table_name
11
+ @polymorphic_settings = polymorphic_settings
12
+ @column_settings = column_settings
13
+ end
14
+
15
+ def column_list
16
+ columns.keys
17
+ end
18
+
19
+ def generate_row(row_number)
20
+ column_list.map { |c| column_data(row_number, c) }
21
+ end
22
+
23
+ private
24
+
25
+ def column_data(row_number, column)
26
+ column_value = columns[column]
27
+ return column_value unless column_value.respond_to?(:call)
28
+
29
+ if column_value.arity == 2
30
+ column_value.call(row_number, column)
31
+ elsif column_value.arity == 1
32
+ column_value.call(row_number)
33
+ else
34
+ column_value.call
35
+ end
36
+ end
37
+
38
+ def columns
39
+ @columns ||= [
40
+ own_columns_config,
41
+ belongs_to_config,
42
+ polymorphic_config,
43
+ @column_settings,
44
+ ].reduce(:merge)
45
+ end
46
+
47
+ def own_columns_config
48
+ @model_class
49
+ .columns_hash
50
+ .reject { |name| name == @model_class.primary_key }
51
+ .select { |_, c| ColumnConfiguration.supported?(model_class: @model_class, ar_column: c) }
52
+ .map { |_, c| ColumnConfiguration.config_for(model_class: @model_class, ar_column: c) }
53
+ .reduce({}, :merge)
54
+ end
55
+
56
+ def belongs_to_config
57
+ @model_class
58
+ .reflect_on_all_associations
59
+ .select(&:belongs_to?)
60
+ .reject(&:polymorphic?)
61
+ .map { |assoc| BelongsToConfiguration.config_for(ar_association: assoc) }
62
+ .reduce({}, :merge)
63
+ end
64
+
65
+ def polymorphic_config
66
+ @polymorphic_settings
67
+ .map { |s| PolymorphicBelongsToConfiguration.config_for(polymorphic_settings: s) }
68
+ .reduce({}, :merge)
69
+ end
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveRecordDataLoader
4
+ module ActiveRecord
5
+ class PolymorphicBelongsToConfiguration
6
+ def self.config_for(polymorphic_settings:)
7
+ ar_association = polymorphic_settings.model_class.reflect_on_association(
8
+ polymorphic_settings.name
9
+ )
10
+ raise "#{name} only supports polymorphic associations" unless ar_association.polymorphic?
11
+
12
+ new(polymorphic_settings, ar_association).polymorphic_config
13
+ end
14
+
15
+ def initialize(settings, ar_association)
16
+ @settings = settings
17
+ @ar_association = ar_association
18
+ @model_count = settings.weighted_models.size
19
+ end
20
+
21
+ def polymorphic_config
22
+ {
23
+ @ar_association.foreign_type.to_sym => ->(row_number) { foreign_type(row_number) },
24
+ @ar_association.foreign_key.to_sym => ->(row_number) { foreign_key(row_number) },
25
+ }
26
+ end
27
+
28
+ private
29
+
30
+ def foreign_type(row_number)
31
+ possible_values[row_number % @model_count][0]
32
+ end
33
+
34
+ def foreign_key(row_number)
35
+ possible_values[row_number % @model_count][1].sample
36
+ end
37
+
38
+ def possible_values
39
+ @possible_values ||= begin
40
+ values = @settings.models.keys.map do |klass|
41
+ [klass.name, klass.all.pluck(klass.primary_key).to_a]
42
+ end.to_h
43
+
44
+ @settings.weighted_models.map { |klass| [klass.name, values[klass.name]] }
45
+ end
46
+ end
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,72 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ActiveRecordDataLoader
4
+ module ActiveRecord
5
+ class TextValueGenerator
6
+ GENERATORS = {
7
+ likely_a_person_full_name?: -> { ActiveRecordDataLoader::DataFaker.person_name },
8
+ likely_a_first_name?: -> { ActiveRecordDataLoader::DataFaker.first_name },
9
+ likely_a_middle_name?: -> { ActiveRecordDataLoader::DataFaker.middle_name },
10
+ likely_a_last_name?: -> { ActiveRecordDataLoader::DataFaker.last_name },
11
+ likely_an_organization_name?: -> { ActiveRecordDataLoader::DataFaker.company_name },
12
+ }.freeze
13
+
14
+ class << self
15
+ def generator_for(model_class:, ar_column:)
16
+ scenario = GENERATORS.keys.find { |m| send(m, model_class, ar_column) }
17
+ generator = GENERATORS.fetch(scenario, -> { SecureRandom.uuid })
18
+
19
+ -> { truncate_if_needed(generator.call, ar_column.limit) }
20
+ end
21
+
22
+ private
23
+
24
+ def truncate_if_needed(value, limit)
25
+ return value if limit.nil?
26
+
27
+ value[0...limit]
28
+ end
29
+
30
+ def likely_a_person_full_name?(model_class, ar_column)
31
+ ar_column.name.downcase == "name" && likely_a_person?(model_class)
32
+ end
33
+
34
+ def likely_a_first_name?(_, ar_column)
35
+ ar_column.name.downcase == "first_name" || ar_column.name.downcase == "firstname"
36
+ end
37
+
38
+ def likely_a_middle_name?(_, ar_column)
39
+ ar_column.name.downcase == "middle_name" || ar_column.name.downcase == "middlename"
40
+ end
41
+
42
+ def likely_a_last_name?(_, ar_column)
43
+ ar_column.name.downcase == "last_name" || ar_column.name.downcase == "lastname"
44
+ end
45
+
46
+ def likely_an_organization_name?(model_class, ar_column)
47
+ ar_column.name.downcase == "company_name" ||
48
+ ar_column.name.downcase == "companyname" ||
49
+ ar_column.name.downcase == "business_name" ||
50
+ ar_column.name.downcase == "businessname" ||
51
+ (ar_column.name.downcase == "name" && likely_an_organization?(model_class))
52
+ end
53
+
54
+ def likely_a_person?(model_class)
55
+ %w[
56
+ customer human employee person user
57
+ ].any? do |word|
58
+ model_class.name.downcase.start_with?(word) || model_class.name.downcase.end_with?(word)
59
+ end
60
+ end
61
+
62
+ def likely_an_organization?(model_class)
63
+ %w[
64
+ business company enterprise legalentity organization
65
+ ].any? do |word|
66
+ model_class.name.downcase.start_with?(word) || model_class.name.downcase.end_with?(word)
67
+ end
68
+ end
69
+ end
70
+ end
71
+ end
72
+ end