active_record_data_loader 1.0.1 → 1.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.travis.yml +1 -0
- data/CHANGELOG.md +8 -0
- data/Gemfile.lock +3 -1
- data/active_record_data_loader.gemspec +1 -0
- data/config/database.yml +7 -0
- data/config/database.yml.travis +5 -0
- data/lib/active_record_data_loader.rb +19 -16
- data/lib/active_record_data_loader/active_record/column_configuration.rb +13 -4
- data/lib/active_record_data_loader/active_record/datetime_value_generator.rb +1 -1
- data/lib/active_record_data_loader/active_record/enum_value_generator.rb +25 -3
- data/lib/active_record_data_loader/active_record/integer_value_generator.rb +1 -1
- data/lib/active_record_data_loader/active_record/model_data_generator.rb +15 -2
- data/lib/active_record_data_loader/active_record/text_value_generator.rb +1 -1
- data/lib/active_record_data_loader/configuration.rb +4 -2
- data/lib/active_record_data_loader/loader.rb +16 -10
- data/lib/active_record_data_loader/version.rb +1 -1
- metadata +18 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: a76332e648af9f10d665eaab3fb442b18d708914
|
4
|
+
data.tar.gz: 012eb43c82d6bca82f97ece0d719faa07ea79c6c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2f41b6aedb4c4cd4f02319cd3d13c50715a0e91bbfd0710e04caa2145054d26b20a4e7a2f69b295b28650a0ad8e111f322fbf008bda28f0db7fa6fa25428de24
|
7
|
+
data.tar.gz: 18d9fb81d1e5d273aab3d21bc2d12f5e28624636f6286deeaf0ccadfdbf20afe629583c96dde78ee32acaaa0b931a363fe32c2c8a77435eca32540875a8d3ee7
|
data/.travis.yml
CHANGED
@@ -19,5 +19,6 @@ notifications:
|
|
19
19
|
before_install: "gem update --system && gem install bundler"
|
20
20
|
before_script:
|
21
21
|
- psql -c 'create database test;' -U postgres
|
22
|
+
- mysql -e 'CREATE DATABASE IF NOT EXISTS test;'
|
22
23
|
- cp config/database.yml.travis config/database.yml
|
23
24
|
script: "bundle exec rake"
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,13 @@
|
|
1
1
|
# Change log
|
2
2
|
|
3
|
+
## [v1.0.2] - 2019-07-05
|
4
|
+
|
5
|
+
[Diff](https://github.com/abeiderman/active_record_data_loader/compare/v1.0.1...v1.0.2)
|
6
|
+
|
7
|
+
### Enhancements:
|
8
|
+
* Add support for MySQL enums
|
9
|
+
* Accept a connection factory lambda as part of the configuration
|
10
|
+
|
3
11
|
## [v1.0.1] - 2019-06-16
|
4
12
|
|
5
13
|
[Diff](https://github.com/abeiderman/active_record_data_loader/compare/v1.0.0...v1.0.1)
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
active_record_data_loader (1.0.
|
4
|
+
active_record_data_loader (1.0.2)
|
5
5
|
activerecord (>= 4.0)
|
6
6
|
|
7
7
|
GEM
|
@@ -40,6 +40,7 @@ GEM
|
|
40
40
|
json (2.2.0)
|
41
41
|
method_source (0.9.2)
|
42
42
|
minitest (5.11.3)
|
43
|
+
mysql2 (0.5.2)
|
43
44
|
parallel (1.17.0)
|
44
45
|
parser (2.6.3.0)
|
45
46
|
ast (~> 2.4.0)
|
@@ -96,6 +97,7 @@ DEPENDENCIES
|
|
96
97
|
appraisal
|
97
98
|
bundler (>= 1.16)
|
98
99
|
coveralls
|
100
|
+
mysql2
|
99
101
|
pg
|
100
102
|
pry
|
101
103
|
rake (~> 12.0)
|
@@ -37,6 +37,7 @@ Gem::Specification.new do |spec|
|
|
37
37
|
spec.add_development_dependency "appraisal"
|
38
38
|
spec.add_development_dependency "bundler", ">= 1.16"
|
39
39
|
spec.add_development_dependency "coveralls"
|
40
|
+
spec.add_development_dependency "mysql2"
|
40
41
|
spec.add_development_dependency "pg"
|
41
42
|
spec.add_development_dependency "pry"
|
42
43
|
spec.add_development_dependency "rake", "~> 12.0"
|
data/config/database.yml
CHANGED
data/config/database.yml.travis
CHANGED
@@ -24,7 +24,7 @@ require "active_record_data_loader/loader"
|
|
24
24
|
module ActiveRecordDataLoader
|
25
25
|
def self.define(config = ActiveRecordDataLoader.configuration, &block)
|
26
26
|
LoaderProxy.new(
|
27
|
-
|
27
|
+
config,
|
28
28
|
ActiveRecordDataLoader::Dsl::Definition.new(config).tap { |l| l.instance_eval(&block) }
|
29
29
|
)
|
30
30
|
end
|
@@ -46,25 +46,28 @@ module ActiveRecordDataLoader
|
|
46
46
|
def load_data
|
47
47
|
ActiveRecordDataLoader::ActiveRecord::PerRowValueCache.clear
|
48
48
|
|
49
|
-
definition.models.map
|
50
|
-
generator = ActiveRecordDataLoader::ActiveRecord::ModelDataGenerator.new(
|
51
|
-
model: m.klass,
|
52
|
-
column_settings: m.columns,
|
53
|
-
polymorphic_settings: m.polymorphic_associations,
|
54
|
-
belongs_to_settings: m.belongs_to_associations
|
55
|
-
)
|
56
|
-
|
57
|
-
ActiveRecordDataLoader::Loader.load_data(
|
58
|
-
data_generator: generator,
|
59
|
-
batch_size: m.batch_size,
|
60
|
-
total_rows: m.row_count,
|
61
|
-
configuration: configuration
|
62
|
-
)
|
63
|
-
end
|
49
|
+
definition.models.map { |m| load_model(m) }
|
64
50
|
end
|
65
51
|
|
66
52
|
private
|
67
53
|
|
68
54
|
attr_reader :definition, :configuration
|
55
|
+
|
56
|
+
def load_model(model)
|
57
|
+
generator = ActiveRecordDataLoader::ActiveRecord::ModelDataGenerator.new(
|
58
|
+
model: model.klass,
|
59
|
+
column_settings: model.columns,
|
60
|
+
polymorphic_settings: model.polymorphic_associations,
|
61
|
+
belongs_to_settings: model.belongs_to_associations,
|
62
|
+
connection_factory: configuration.connection_factory
|
63
|
+
)
|
64
|
+
|
65
|
+
ActiveRecordDataLoader::Loader.load_data(
|
66
|
+
data_generator: generator,
|
67
|
+
batch_size: model.batch_size,
|
68
|
+
total_rows: model.row_count,
|
69
|
+
configuration: configuration
|
70
|
+
)
|
71
|
+
end
|
69
72
|
end
|
70
73
|
end
|
@@ -12,13 +12,14 @@ module ActiveRecordDataLoader
|
|
12
12
|
datetime: DatetimeValueGenerator,
|
13
13
|
}.freeze
|
14
14
|
|
15
|
-
def config_for(model_class:, ar_column:)
|
15
|
+
def config_for(model_class:, ar_column:, connection_factory:)
|
16
16
|
raise_error_if_not_supported(model_class, ar_column)
|
17
17
|
|
18
18
|
{
|
19
|
-
ar_column.name.to_sym => VALUE_GENERATORS[ar_column
|
19
|
+
ar_column.name.to_sym => VALUE_GENERATORS[column_type(ar_column)].generator_for(
|
20
20
|
model_class: model_class,
|
21
|
-
ar_column: ar_column
|
21
|
+
ar_column: ar_column,
|
22
|
+
connection_factory: connection_factory
|
22
23
|
),
|
23
24
|
}
|
24
25
|
end
|
@@ -26,7 +27,7 @@ module ActiveRecordDataLoader
|
|
26
27
|
def supported?(model_class:, ar_column:)
|
27
28
|
return false if model_class.reflect_on_association(ar_column.name)
|
28
29
|
|
29
|
-
VALUE_GENERATORS.keys.include?(ar_column
|
30
|
+
VALUE_GENERATORS.keys.include?(column_type(ar_column))
|
30
31
|
end
|
31
32
|
|
32
33
|
private
|
@@ -38,6 +39,14 @@ module ActiveRecordDataLoader
|
|
38
39
|
Column '#{ar_column.name}' of type '#{ar_column.type}' in model '#{model_class.name}' not supported"
|
39
40
|
ERROR
|
40
41
|
end
|
42
|
+
|
43
|
+
def column_type(ar_column)
|
44
|
+
if ar_column.type == :string && ar_column.sql_type.to_s.downcase.start_with?("enum")
|
45
|
+
:enum
|
46
|
+
else
|
47
|
+
ar_column.type
|
48
|
+
end
|
49
|
+
end
|
41
50
|
end
|
42
51
|
end
|
43
52
|
end
|
@@ -4,14 +4,26 @@ module ActiveRecordDataLoader
|
|
4
4
|
module ActiveRecord
|
5
5
|
class EnumValueGenerator
|
6
6
|
class << self
|
7
|
-
def generator_for(model_class:, ar_column:)
|
8
|
-
values = enum_values_for(model_class, ar_column.sql_type)
|
7
|
+
def generator_for(model_class:, ar_column:, connection_factory:)
|
8
|
+
values = enum_values_for(model_class, ar_column.sql_type, connection_factory)
|
9
9
|
-> { values.sample }
|
10
10
|
end
|
11
11
|
|
12
12
|
private
|
13
13
|
|
14
|
-
def enum_values_for(model_class, enum_type)
|
14
|
+
def enum_values_for(model_class, enum_type, connection_factory)
|
15
|
+
connection = connection_factory.call
|
16
|
+
|
17
|
+
if connection.adapter_name.downcase.to_sym == :postgresql
|
18
|
+
postgres_enum_values_for(model_class, enum_type)
|
19
|
+
elsif connection.adapter_name.downcase.to_s.start_with?("mysql")
|
20
|
+
mysql_enum_values_for(model_class, enum_type)
|
21
|
+
else
|
22
|
+
[]
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def postgres_enum_values_for(model_class, enum_type)
|
15
27
|
model_class
|
16
28
|
.connection
|
17
29
|
.execute("SELECT unnest(enum_range(NULL::#{enum_type}))::text")
|
@@ -19,6 +31,16 @@ module ActiveRecordDataLoader
|
|
19
31
|
.flatten
|
20
32
|
.compact
|
21
33
|
end
|
34
|
+
|
35
|
+
def mysql_enum_values_for(_model_class, enum_type)
|
36
|
+
enum_type
|
37
|
+
.to_s
|
38
|
+
.downcase
|
39
|
+
.gsub(/\Aenum\(|\)\Z/, "")
|
40
|
+
.split(",")
|
41
|
+
.map(&:strip)
|
42
|
+
.map { |s| s.gsub(/\A'|'\Z/, "") }
|
43
|
+
end
|
22
44
|
end
|
23
45
|
end
|
24
46
|
end
|
@@ -4,7 +4,7 @@ module ActiveRecordDataLoader
|
|
4
4
|
module ActiveRecord
|
5
5
|
class IntegerValueGenerator
|
6
6
|
class << self
|
7
|
-
def generator_for(model_class:, ar_column:)
|
7
|
+
def generator_for(model_class:, ar_column:, connection_factory: nil)
|
8
8
|
range_limit = [(256**number_of_bytes(ar_column)) / 2 - 1, 1_000_000_000].min
|
9
9
|
|
10
10
|
-> { rand(0..range_limit) }
|
@@ -5,12 +5,19 @@ module ActiveRecordDataLoader
|
|
5
5
|
class ModelDataGenerator
|
6
6
|
attr_reader :table
|
7
7
|
|
8
|
-
def initialize(
|
8
|
+
def initialize(
|
9
|
+
model:,
|
10
|
+
column_settings:,
|
11
|
+
polymorphic_settings: [],
|
12
|
+
belongs_to_settings: [],
|
13
|
+
connection_factory:
|
14
|
+
)
|
9
15
|
@model_class = model
|
10
16
|
@table = model.table_name
|
11
17
|
@column_settings = column_settings
|
12
18
|
@polymorphic_settings = polymorphic_settings
|
13
19
|
@belongs_to_settings = belongs_to_settings.map { |s| [s.name, s.query] }.to_h
|
20
|
+
@connection_factory = connection_factory
|
14
21
|
end
|
15
22
|
|
16
23
|
def column_list
|
@@ -50,7 +57,13 @@ module ActiveRecordDataLoader
|
|
50
57
|
.columns_hash
|
51
58
|
.reject { |name| name == @model_class.primary_key }
|
52
59
|
.select { |_, c| ColumnConfiguration.supported?(model_class: @model_class, ar_column: c) }
|
53
|
-
.map
|
60
|
+
.map do |_, c|
|
61
|
+
ColumnConfiguration.config_for(
|
62
|
+
model_class: @model_class,
|
63
|
+
ar_column: c,
|
64
|
+
connection_factory: @connection_factory
|
65
|
+
)
|
66
|
+
end
|
54
67
|
.reduce({}, :merge)
|
55
68
|
end
|
56
69
|
|
@@ -12,7 +12,7 @@ module ActiveRecordDataLoader
|
|
12
12
|
}.freeze
|
13
13
|
|
14
14
|
class << self
|
15
|
-
def generator_for(model_class:, ar_column:)
|
15
|
+
def generator_for(model_class:, ar_column:, connection_factory: nil)
|
16
16
|
scenario = GENERATORS.keys.find { |m| send(m, model_class, ar_column) }
|
17
17
|
generator = GENERATORS.fetch(scenario, -> { SecureRandom.uuid })
|
18
18
|
|
@@ -2,18 +2,20 @@
|
|
2
2
|
|
3
3
|
module ActiveRecordDataLoader
|
4
4
|
class Configuration
|
5
|
-
attr_accessor :default_batch_size, :default_row_count, :logger, :statement_timeout
|
5
|
+
attr_accessor :default_batch_size, :default_row_count, :logger, :statement_timeout, :connection_factory
|
6
6
|
|
7
7
|
def initialize(
|
8
8
|
default_batch_size: 100_000,
|
9
9
|
default_row_count: 1,
|
10
10
|
logger: nil,
|
11
|
-
statement_timeout: "2min"
|
11
|
+
statement_timeout: "2min",
|
12
|
+
connection_factory: -> { ::ActiveRecord::Base.connection }
|
12
13
|
)
|
13
14
|
@default_batch_size = default_batch_size
|
14
15
|
@default_row_count = default_row_count
|
15
16
|
@logger = logger || default_logger
|
16
17
|
@statement_timeout = statement_timeout
|
18
|
+
@connection_factory = connection_factory
|
17
19
|
end
|
18
20
|
|
19
21
|
private
|
@@ -14,14 +14,15 @@ module ActiveRecordDataLoader
|
|
14
14
|
new(
|
15
15
|
logger: configuration.logger,
|
16
16
|
statement_timeout: configuration.statement_timeout,
|
17
|
-
strategy: strategy_class.new(data_generator)
|
17
|
+
strategy: strategy_class(configuration.connection_factory).new(data_generator),
|
18
|
+
connection_factory: configuration.connection_factory
|
18
19
|
).load_data(batch_size, total_rows)
|
19
20
|
end
|
20
21
|
|
21
22
|
private
|
22
23
|
|
23
|
-
def strategy_class
|
24
|
-
if
|
24
|
+
def strategy_class(connection_factory)
|
25
|
+
if connection_factory.call.raw_connection.respond_to?(:copy_data)
|
25
26
|
ActiveRecordDataLoader::CopyStrategy
|
26
27
|
else
|
27
28
|
ActiveRecordDataLoader::BulkInsertStrategy
|
@@ -29,10 +30,11 @@ module ActiveRecordDataLoader
|
|
29
30
|
end
|
30
31
|
end
|
31
32
|
|
32
|
-
def initialize(logger:, statement_timeout:, strategy:)
|
33
|
+
def initialize(logger:, statement_timeout:, strategy:, connection_factory:)
|
33
34
|
@logger = logger
|
34
35
|
@strategy = strategy
|
35
36
|
@statement_timeout = statement_timeout
|
37
|
+
@connection_factory = connection_factory
|
36
38
|
end
|
37
39
|
|
38
40
|
def load_data(batch_size, total_rows)
|
@@ -55,7 +57,7 @@ module ActiveRecordDataLoader
|
|
55
57
|
|
56
58
|
private
|
57
59
|
|
58
|
-
attr_reader :strategy, :statement_timeout, :logger
|
60
|
+
attr_reader :strategy, :statement_timeout, :logger, :connection_factory
|
59
61
|
|
60
62
|
def load_in_batches(batch_size, total_rows, batch_count)
|
61
63
|
with_connection do |connection|
|
@@ -71,22 +73,26 @@ module ActiveRecordDataLoader
|
|
71
73
|
end
|
72
74
|
|
73
75
|
def with_connection
|
74
|
-
if
|
76
|
+
if connection.adapter_name.downcase.to_sym == :postgresql
|
75
77
|
original_timeout = retrieve_statement_timeout
|
76
78
|
update_statement_timeout(statement_timeout)
|
77
|
-
yield
|
79
|
+
yield connection
|
78
80
|
update_statement_timeout(original_timeout)
|
79
81
|
else
|
80
|
-
yield
|
82
|
+
yield connection
|
81
83
|
end
|
82
84
|
end
|
83
85
|
|
84
86
|
def retrieve_statement_timeout
|
85
|
-
|
87
|
+
connection.execute("SHOW statement_timeout").first["statement_timeout"]
|
86
88
|
end
|
87
89
|
|
88
90
|
def update_statement_timeout(timeout)
|
89
|
-
|
91
|
+
connection.execute("SET statement_timeout = \"#{timeout}\"")
|
92
|
+
end
|
93
|
+
|
94
|
+
def connection
|
95
|
+
connection_factory.call
|
90
96
|
end
|
91
97
|
end
|
92
98
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: active_record_data_loader
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alejandro Beiderman
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-06
|
11
|
+
date: 2019-07-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -66,6 +66,20 @@ dependencies:
|
|
66
66
|
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: mysql2
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
69
83
|
- !ruby/object:Gem::Dependency
|
70
84
|
name: pg
|
71
85
|
requirement: !ruby/object:Gem::Requirement
|
@@ -251,7 +265,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
251
265
|
- !ruby/object:Gem::Version
|
252
266
|
version: '0'
|
253
267
|
requirements: []
|
254
|
-
|
268
|
+
rubyforge_project:
|
269
|
+
rubygems_version: 2.6.14
|
255
270
|
signing_key:
|
256
271
|
specification_version: 4
|
257
272
|
summary: A utility to bulk load test data for performance testing.
|