active_record_data_loader 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +13 -0
- data/.rspec +3 -0
- data/.rubocop.yml +38 -0
- data/.travis.yml +11 -0
- data/Appraisals +17 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/Gemfile +8 -0
- data/Gemfile.lock +107 -0
- data/LICENSE.txt +21 -0
- data/README.md +49 -0
- data/Rakefile +10 -0
- data/active_record_data_loader.gemspec +47 -0
- data/bin/console +15 -0
- data/bin/setup +8 -0
- data/config/database.yml +10 -0
- data/config/database.yml.travis +7 -0
- data/gemfiles/.bundle/config +2 -0
- data/gemfiles/activerecord_5.gemfile +7 -0
- data/gemfiles/activerecord_6.gemfile +7 -0
- data/gemfiles/faker.gemfile +7 -0
- data/gemfiles/ffaker.gemfile +7 -0
- data/lib/active_record_data_loader.rb +64 -0
- data/lib/active_record_data_loader/active_record/belongs_to_configuration.rb +27 -0
- data/lib/active_record_data_loader/active_record/column_configuration.rb +43 -0
- data/lib/active_record_data_loader/active_record/enum_value_generator.rb +25 -0
- data/lib/active_record_data_loader/active_record/integer_value_generator.rb +21 -0
- data/lib/active_record_data_loader/active_record/model_data_generator.rb +72 -0
- data/lib/active_record_data_loader/active_record/polymorphic_belongs_to_configuration.rb +49 -0
- data/lib/active_record_data_loader/active_record/text_value_generator.rb +72 -0
- data/lib/active_record_data_loader/bulk_insert_strategy.rb +57 -0
- data/lib/active_record_data_loader/configuration.rb +17 -0
- data/lib/active_record_data_loader/copy_strategy.rb +58 -0
- data/lib/active_record_data_loader/data_faker.rb +85 -0
- data/lib/active_record_data_loader/dsl/definition.rb +25 -0
- data/lib/active_record_data_loader/dsl/model.rb +35 -0
- data/lib/active_record_data_loader/dsl/polymorphic_association.rb +25 -0
- data/lib/active_record_data_loader/loader.rb +66 -0
- data/lib/active_record_data_loader/version.rb +5 -0
- data/script/ci_build.sh +6 -0
- metadata +240 -0
data/config/database.yml
ADDED
@@ -0,0 +1,64 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "active_record_data_loader/version"
|
4
|
+
require "active_record"
|
5
|
+
require "active_record_data_loader/configuration"
|
6
|
+
require "active_record_data_loader/data_faker"
|
7
|
+
require "active_record_data_loader/active_record/integer_value_generator"
|
8
|
+
require "active_record_data_loader/active_record/text_value_generator"
|
9
|
+
require "active_record_data_loader/active_record/enum_value_generator"
|
10
|
+
require "active_record_data_loader/active_record/column_configuration"
|
11
|
+
require "active_record_data_loader/active_record/belongs_to_configuration"
|
12
|
+
require "active_record_data_loader/active_record/polymorphic_belongs_to_configuration"
|
13
|
+
require "active_record_data_loader/active_record/model_data_generator"
|
14
|
+
require "active_record_data_loader/dsl/polymorphic_association"
|
15
|
+
require "active_record_data_loader/dsl/model"
|
16
|
+
require "active_record_data_loader/dsl/definition"
|
17
|
+
require "active_record_data_loader/copy_strategy"
|
18
|
+
require "active_record_data_loader/bulk_insert_strategy"
|
19
|
+
require "active_record_data_loader/loader"
|
20
|
+
|
21
|
+
module ActiveRecordDataLoader
|
22
|
+
def self.define(config = ActiveRecordDataLoader.configuration, &block)
|
23
|
+
LoaderProxy.new(
|
24
|
+
configuration,
|
25
|
+
ActiveRecordDataLoader::Dsl::Definition.new(config).tap { |l| l.instance_eval(&block) }
|
26
|
+
)
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.configure(&block)
|
30
|
+
@configuration = ActiveRecordDataLoader::Configuration.new.tap { |c| block.call(c) }
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.configuration
|
34
|
+
@configuration ||= ActiveRecordDataLoader::Configuration.new
|
35
|
+
end
|
36
|
+
|
37
|
+
class LoaderProxy
|
38
|
+
def initialize(configuration, definition)
|
39
|
+
@configuration = configuration
|
40
|
+
@definition = definition
|
41
|
+
end
|
42
|
+
|
43
|
+
def load_data
|
44
|
+
definition.models.map do |m|
|
45
|
+
generator = ActiveRecordDataLoader::ActiveRecord::ModelDataGenerator.new(
|
46
|
+
model: m.klass,
|
47
|
+
column_settings: m.columns,
|
48
|
+
polymorphic_settings: m.polymorphic_associations
|
49
|
+
)
|
50
|
+
|
51
|
+
ActiveRecordDataLoader::Loader.load_data(
|
52
|
+
data_generator: generator,
|
53
|
+
batch_size: m.batch_size,
|
54
|
+
total_rows: m.row_count,
|
55
|
+
logger: configuration.logger
|
56
|
+
)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
private
|
61
|
+
|
62
|
+
attr_reader :definition, :configuration
|
63
|
+
end
|
64
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ActiveRecordDataLoader
|
4
|
+
module ActiveRecord
|
5
|
+
class BelongsToConfiguration
|
6
|
+
def self.config_for(ar_association:)
|
7
|
+
raise "#{name} does not support polymorphic associations" if ar_association.polymorphic?
|
8
|
+
|
9
|
+
{ ar_association.join_foreign_key.to_sym => new(ar_association).foreign_key_func }
|
10
|
+
end
|
11
|
+
|
12
|
+
def initialize(ar_association)
|
13
|
+
@ar_association = ar_association
|
14
|
+
end
|
15
|
+
|
16
|
+
def foreign_key_func
|
17
|
+
-> { possible_values.sample }
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
|
22
|
+
def possible_values
|
23
|
+
@possible_values ||= @ar_association.klass.all.pluck(@ar_association.join_primary_key).to_a
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ActiveRecordDataLoader
|
4
|
+
module ActiveRecord
|
5
|
+
class ColumnConfiguration
|
6
|
+
class << self
|
7
|
+
VALUE_GENERATORS = {
|
8
|
+
enum: EnumValueGenerator,
|
9
|
+
integer: IntegerValueGenerator,
|
10
|
+
string: TextValueGenerator,
|
11
|
+
text: TextValueGenerator,
|
12
|
+
}.freeze
|
13
|
+
|
14
|
+
def config_for(model_class:, ar_column:)
|
15
|
+
raise_error_if_not_supported(model_class, ar_column)
|
16
|
+
|
17
|
+
{
|
18
|
+
ar_column.name.to_sym => VALUE_GENERATORS[ar_column.type].generator_for(
|
19
|
+
model_class: model_class,
|
20
|
+
ar_column: ar_column
|
21
|
+
),
|
22
|
+
}
|
23
|
+
end
|
24
|
+
|
25
|
+
def supported?(model_class:, ar_column:)
|
26
|
+
return false if model_class.reflect_on_association(ar_column.name)
|
27
|
+
|
28
|
+
VALUE_GENERATORS.keys.include?(ar_column.type)
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def raise_error_if_not_supported(model_class, ar_column)
|
34
|
+
return if supported?(model_class: model_class, ar_column: ar_column)
|
35
|
+
|
36
|
+
raise <<~ERROR
|
37
|
+
Column '#{ar_column.name}' of type '#{ar_column.type}' in model '#{model_class.name}' not supported"
|
38
|
+
ERROR
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ActiveRecordDataLoader
|
4
|
+
module ActiveRecord
|
5
|
+
class EnumValueGenerator
|
6
|
+
class << self
|
7
|
+
def generator_for(model_class:, ar_column:)
|
8
|
+
values = enum_values_for(model_class, ar_column.sql_type)
|
9
|
+
-> { values.sample }
|
10
|
+
end
|
11
|
+
|
12
|
+
private
|
13
|
+
|
14
|
+
def enum_values_for(model_class, enum_type)
|
15
|
+
model_class
|
16
|
+
.connection
|
17
|
+
.execute("SELECT unnest(enum_range(NULL::#{enum_type}))::text")
|
18
|
+
.map(&:values)
|
19
|
+
.flatten
|
20
|
+
.compact
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ActiveRecordDataLoader
|
4
|
+
module ActiveRecord
|
5
|
+
class IntegerValueGenerator
|
6
|
+
class << self
|
7
|
+
def generator_for(model_class:, ar_column:)
|
8
|
+
range_limit = [(256**number_of_bytes(ar_column)) / 2 - 1, 1_000_000_000].min
|
9
|
+
|
10
|
+
-> { rand(0..range_limit) }
|
11
|
+
end
|
12
|
+
|
13
|
+
private
|
14
|
+
|
15
|
+
def number_of_bytes(ar_column)
|
16
|
+
ar_column.limit || 8
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ActiveRecordDataLoader
|
4
|
+
module ActiveRecord
|
5
|
+
class ModelDataGenerator
|
6
|
+
attr_reader :table
|
7
|
+
|
8
|
+
def initialize(model:, column_settings:, polymorphic_settings: [])
|
9
|
+
@model_class = model
|
10
|
+
@table = model.table_name
|
11
|
+
@polymorphic_settings = polymorphic_settings
|
12
|
+
@column_settings = column_settings
|
13
|
+
end
|
14
|
+
|
15
|
+
def column_list
|
16
|
+
columns.keys
|
17
|
+
end
|
18
|
+
|
19
|
+
def generate_row(row_number)
|
20
|
+
column_list.map { |c| column_data(row_number, c) }
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def column_data(row_number, column)
|
26
|
+
column_value = columns[column]
|
27
|
+
return column_value unless column_value.respond_to?(:call)
|
28
|
+
|
29
|
+
if column_value.arity == 2
|
30
|
+
column_value.call(row_number, column)
|
31
|
+
elsif column_value.arity == 1
|
32
|
+
column_value.call(row_number)
|
33
|
+
else
|
34
|
+
column_value.call
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def columns
|
39
|
+
@columns ||= [
|
40
|
+
own_columns_config,
|
41
|
+
belongs_to_config,
|
42
|
+
polymorphic_config,
|
43
|
+
@column_settings,
|
44
|
+
].reduce(:merge)
|
45
|
+
end
|
46
|
+
|
47
|
+
def own_columns_config
|
48
|
+
@model_class
|
49
|
+
.columns_hash
|
50
|
+
.reject { |name| name == @model_class.primary_key }
|
51
|
+
.select { |_, c| ColumnConfiguration.supported?(model_class: @model_class, ar_column: c) }
|
52
|
+
.map { |_, c| ColumnConfiguration.config_for(model_class: @model_class, ar_column: c) }
|
53
|
+
.reduce({}, :merge)
|
54
|
+
end
|
55
|
+
|
56
|
+
def belongs_to_config
|
57
|
+
@model_class
|
58
|
+
.reflect_on_all_associations
|
59
|
+
.select(&:belongs_to?)
|
60
|
+
.reject(&:polymorphic?)
|
61
|
+
.map { |assoc| BelongsToConfiguration.config_for(ar_association: assoc) }
|
62
|
+
.reduce({}, :merge)
|
63
|
+
end
|
64
|
+
|
65
|
+
def polymorphic_config
|
66
|
+
@polymorphic_settings
|
67
|
+
.map { |s| PolymorphicBelongsToConfiguration.config_for(polymorphic_settings: s) }
|
68
|
+
.reduce({}, :merge)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ActiveRecordDataLoader
|
4
|
+
module ActiveRecord
|
5
|
+
class PolymorphicBelongsToConfiguration
|
6
|
+
def self.config_for(polymorphic_settings:)
|
7
|
+
ar_association = polymorphic_settings.model_class.reflect_on_association(
|
8
|
+
polymorphic_settings.name
|
9
|
+
)
|
10
|
+
raise "#{name} only supports polymorphic associations" unless ar_association.polymorphic?
|
11
|
+
|
12
|
+
new(polymorphic_settings, ar_association).polymorphic_config
|
13
|
+
end
|
14
|
+
|
15
|
+
def initialize(settings, ar_association)
|
16
|
+
@settings = settings
|
17
|
+
@ar_association = ar_association
|
18
|
+
@model_count = settings.weighted_models.size
|
19
|
+
end
|
20
|
+
|
21
|
+
def polymorphic_config
|
22
|
+
{
|
23
|
+
@ar_association.foreign_type.to_sym => ->(row_number) { foreign_type(row_number) },
|
24
|
+
@ar_association.foreign_key.to_sym => ->(row_number) { foreign_key(row_number) },
|
25
|
+
}
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def foreign_type(row_number)
|
31
|
+
possible_values[row_number % @model_count][0]
|
32
|
+
end
|
33
|
+
|
34
|
+
def foreign_key(row_number)
|
35
|
+
possible_values[row_number % @model_count][1].sample
|
36
|
+
end
|
37
|
+
|
38
|
+
def possible_values
|
39
|
+
@possible_values ||= begin
|
40
|
+
values = @settings.models.keys.map do |klass|
|
41
|
+
[klass.name, klass.all.pluck(klass.primary_key).to_a]
|
42
|
+
end.to_h
|
43
|
+
|
44
|
+
@settings.weighted_models.map { |klass| [klass.name, values[klass.name]] }
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ActiveRecordDataLoader
|
4
|
+
module ActiveRecord
|
5
|
+
class TextValueGenerator
|
6
|
+
GENERATORS = {
|
7
|
+
likely_a_person_full_name?: -> { ActiveRecordDataLoader::DataFaker.person_name },
|
8
|
+
likely_a_first_name?: -> { ActiveRecordDataLoader::DataFaker.first_name },
|
9
|
+
likely_a_middle_name?: -> { ActiveRecordDataLoader::DataFaker.middle_name },
|
10
|
+
likely_a_last_name?: -> { ActiveRecordDataLoader::DataFaker.last_name },
|
11
|
+
likely_an_organization_name?: -> { ActiveRecordDataLoader::DataFaker.company_name },
|
12
|
+
}.freeze
|
13
|
+
|
14
|
+
class << self
|
15
|
+
def generator_for(model_class:, ar_column:)
|
16
|
+
scenario = GENERATORS.keys.find { |m| send(m, model_class, ar_column) }
|
17
|
+
generator = GENERATORS.fetch(scenario, -> { SecureRandom.uuid })
|
18
|
+
|
19
|
+
-> { truncate_if_needed(generator.call, ar_column.limit) }
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def truncate_if_needed(value, limit)
|
25
|
+
return value if limit.nil?
|
26
|
+
|
27
|
+
value[0...limit]
|
28
|
+
end
|
29
|
+
|
30
|
+
def likely_a_person_full_name?(model_class, ar_column)
|
31
|
+
ar_column.name.downcase == "name" && likely_a_person?(model_class)
|
32
|
+
end
|
33
|
+
|
34
|
+
def likely_a_first_name?(_, ar_column)
|
35
|
+
ar_column.name.downcase == "first_name" || ar_column.name.downcase == "firstname"
|
36
|
+
end
|
37
|
+
|
38
|
+
def likely_a_middle_name?(_, ar_column)
|
39
|
+
ar_column.name.downcase == "middle_name" || ar_column.name.downcase == "middlename"
|
40
|
+
end
|
41
|
+
|
42
|
+
def likely_a_last_name?(_, ar_column)
|
43
|
+
ar_column.name.downcase == "last_name" || ar_column.name.downcase == "lastname"
|
44
|
+
end
|
45
|
+
|
46
|
+
def likely_an_organization_name?(model_class, ar_column)
|
47
|
+
ar_column.name.downcase == "company_name" ||
|
48
|
+
ar_column.name.downcase == "companyname" ||
|
49
|
+
ar_column.name.downcase == "business_name" ||
|
50
|
+
ar_column.name.downcase == "businessname" ||
|
51
|
+
(ar_column.name.downcase == "name" && likely_an_organization?(model_class))
|
52
|
+
end
|
53
|
+
|
54
|
+
def likely_a_person?(model_class)
|
55
|
+
%w[
|
56
|
+
customer human employee person user
|
57
|
+
].any? do |word|
|
58
|
+
model_class.name.downcase.start_with?(word) || model_class.name.downcase.end_with?(word)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def likely_an_organization?(model_class)
|
63
|
+
%w[
|
64
|
+
business company enterprise legalentity organization
|
65
|
+
].any? do |word|
|
66
|
+
model_class.name.downcase.start_with?(word) || model_class.name.downcase.end_with?(word)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|