activerecord-import 0.24.0 → 0.25.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +20 -11
- data/CHANGELOG.md +14 -0
- data/Rakefile +1 -0
- data/gemfiles/5.0.gemfile +1 -0
- data/gemfiles/5.1.gemfile +1 -0
- data/gemfiles/5.2.gemfile +2 -2
- data/lib/activerecord-import.rb +1 -0
- data/lib/activerecord-import/base.rb +1 -0
- data/lib/activerecord-import/import.rb +49 -39
- data/lib/activerecord-import/version.rb +1 -1
- data/test/adapters/makara_postgis.rb +1 -0
- data/test/makara_postgis/import_test.rb +8 -0
- data/test/schema/mysql2_schema.rb +19 -0
- data/test/support/mysql/import_examples.rb +12 -0
- data/test/test_helper.rb +8 -0
- metadata +8 -4
- data/test/schema/mysql_schema.rb +0 -16
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 26224d6c0d4c23f3edaf1faea68466f18c03fbbc
|
4
|
+
data.tar.gz: 55107f77211b86e3be77b287d224e488dbb280ac
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c0815961332178cf2858b39663f1d9f6ef3d61985cd4cb528cef410714e4bf07c354515b5e06124ad4bb5c7e57e7a179c48d7fb5c5e28886bc9e30c4399e82c9
|
7
|
+
data.tar.gz: df5e8ccd8a658d4981cd26167e09fa8c5e0425c58e4397c48e6f7721221d1de3cc4c6cdf903e3330f07cf6a74b93214cd11cca977f6d75cf6e8b43409807c23b
|
data/.travis.yml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
language: ruby
|
2
2
|
cache: bundler
|
3
3
|
rvm:
|
4
|
-
- 2.3.
|
4
|
+
- 2.3.7
|
5
5
|
|
6
6
|
env:
|
7
7
|
global:
|
@@ -20,8 +20,6 @@ matrix:
|
|
20
20
|
include:
|
21
21
|
- rvm: jruby-9.1.14.0
|
22
22
|
env: AR_VERSION=4.2
|
23
|
-
before_install:
|
24
|
-
- gem update --system
|
25
23
|
|
26
24
|
script:
|
27
25
|
- bundle exec rake test:jdbcsqlite3
|
@@ -30,23 +28,32 @@ matrix:
|
|
30
28
|
|
31
29
|
fast_finish: true
|
32
30
|
|
33
|
-
before_script:
|
34
|
-
- mysql -e 'create database activerecord_import_test;'
|
35
|
-
- psql -c 'create database activerecord_import_test;' -U postgres
|
36
|
-
- psql activerecord_import_test -c 'create extension if not exists hstore;' -U postgres
|
37
|
-
- psql -c 'create extension if not exists postgis;' -U postgres
|
38
|
-
- psql -c 'create extension if not exists "uuid-ossp";' -U postgres
|
39
|
-
- cp test/travis/database.yml test/database.yml
|
40
|
-
|
41
31
|
addons:
|
42
32
|
postgresql: "9.5"
|
43
33
|
apt:
|
44
34
|
sources:
|
45
35
|
- travis-ci/sqlite3
|
36
|
+
- mysql-5.7-trusty
|
46
37
|
packages:
|
47
38
|
- sqlite3
|
39
|
+
- mysql-server
|
40
|
+
- mysql-client
|
48
41
|
- postgresql-9.5-postgis-2.3
|
49
42
|
|
43
|
+
before_install:
|
44
|
+
- gem update --system
|
45
|
+
- sudo mysql -e "use mysql; update user set authentication_string=PASSWORD('') where User='root'; update user set plugin='mysql_native_password';FLUSH PRIVILEGES;"
|
46
|
+
- sudo mysql_upgrade
|
47
|
+
- sudo service mysql restart
|
48
|
+
|
49
|
+
before_script:
|
50
|
+
- mysql -e 'create database activerecord_import_test;'
|
51
|
+
- psql -c 'create database activerecord_import_test;' -U postgres
|
52
|
+
- psql activerecord_import_test -c 'create extension if not exists hstore;' -U postgres
|
53
|
+
- psql -c 'create extension if not exists postgis;' -U postgres
|
54
|
+
- psql -c 'create extension if not exists "uuid-ossp";' -U postgres
|
55
|
+
- cp test/travis/database.yml test/database.yml
|
56
|
+
|
50
57
|
script:
|
51
58
|
- bundle exec rake test:mysql2
|
52
59
|
- bundle exec rake test:mysql2_makara
|
@@ -60,3 +67,5 @@ script:
|
|
60
67
|
- bundle exec rubocop
|
61
68
|
|
62
69
|
dist: trusty
|
70
|
+
|
71
|
+
sudo: required
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,17 @@
|
|
1
|
+
## Changes in 0.25.0
|
2
|
+
|
3
|
+
### New Features
|
4
|
+
|
5
|
+
* Add support for makara_postgis adapter. Thanks to @chadwilken via \#527.
|
6
|
+
* Skip validating presence of belongs_to associations. Thanks to @Sohair63, @naiyt, @jkowens via \#528.
|
7
|
+
|
8
|
+
### Fixes
|
9
|
+
|
10
|
+
* Add missing require for ActiveSupport.on_load. Thanks to @onk via \#529.
|
11
|
+
* Support setting attribute values in before_validation callbacks.
|
12
|
+
Thanks to @SirRawlins, @jkowens via \#531.
|
13
|
+
* Ignore virtual columns. Thanks to @dbourguignon, @jkowens via \#530.
|
14
|
+
|
1
15
|
## Changes in 0.24.0
|
2
16
|
|
3
17
|
### Fixes
|
data/Rakefile
CHANGED
data/gemfiles/5.0.gemfile
CHANGED
data/gemfiles/5.1.gemfile
CHANGED
data/gemfiles/5.2.gemfile
CHANGED
@@ -1,2 +1,2 @@
|
|
1
|
-
gem 'activerecord', '~> 5.2.0
|
2
|
-
gem 'composite_primary_keys', '~> 11.0
|
1
|
+
gem 'activerecord', '~> 5.2.0'
|
2
|
+
gem 'composite_primary_keys', '~> 11.0'
|
data/lib/activerecord-import.rb
CHANGED
@@ -38,8 +38,14 @@ module ActiveRecord::Import #:nodoc:
|
|
38
38
|
model.errors.clear
|
39
39
|
|
40
40
|
validate_callbacks = model._validate_callbacks.dup
|
41
|
+
associations = model.class.reflect_on_all_associations(:belongs_to).map(&:name)
|
42
|
+
|
41
43
|
model._validate_callbacks.each do |callback|
|
42
|
-
|
44
|
+
filter = callback.raw_filter
|
45
|
+
if filter.is_a?(ActiveRecord::Validations::UniquenessValidator) ||
|
46
|
+
(defined?(ActiveRecord::Validations::PresenceValidator) && filter.is_a?(ActiveRecord::Validations::PresenceValidator) && associations.include?(filter.attributes.first))
|
47
|
+
validate_callbacks.delete(callback)
|
48
|
+
end
|
43
49
|
end
|
44
50
|
|
45
51
|
model.run_callbacks(:validation) do
|
@@ -95,7 +101,12 @@ class ActiveRecord::Associations::CollectionAssociation
|
|
95
101
|
|
96
102
|
model_klass = reflection.klass
|
97
103
|
symbolized_foreign_key = reflection.foreign_key.to_sym
|
98
|
-
|
104
|
+
|
105
|
+
symbolized_column_names = if model_klass.connection.respond_to?(:supports_virtual_columns?) && model_klass.connection.supports_virtual_columns?
|
106
|
+
model_klass.columns.reject(&:virtual?).map { |c| c.name.to_sym }
|
107
|
+
else
|
108
|
+
model_klass.column_names.map(&:to_sym)
|
109
|
+
end
|
99
110
|
|
100
111
|
owner_primary_key = reflection.active_record_primary_key.to_sym
|
101
112
|
owner_primary_key_value = owner.send(owner_primary_key)
|
@@ -494,8 +505,8 @@ class ActiveRecord::Base
|
|
494
505
|
end
|
495
506
|
end
|
496
507
|
|
497
|
-
is_validating = options[:validate]
|
498
|
-
|
508
|
+
is_validating = options[:validate_with_context].present? ? true : options[:validate]
|
509
|
+
validator = ActiveRecord::Import::Validator.new(options)
|
499
510
|
|
500
511
|
# assume array of model objects
|
501
512
|
if args.last.is_a?( Array ) && args.last.first.is_a?(ActiveRecord::Base)
|
@@ -504,7 +515,11 @@ class ActiveRecord::Base
|
|
504
515
|
column_names = args.first.dup
|
505
516
|
else
|
506
517
|
models = args.first
|
507
|
-
column_names =
|
518
|
+
column_names = if connection.respond_to?(:supports_virtual_columns?) && connection.supports_virtual_columns?
|
519
|
+
columns.reject(&:virtual?).map(&:name)
|
520
|
+
else
|
521
|
+
self.column_names.dup
|
522
|
+
end
|
508
523
|
end
|
509
524
|
|
510
525
|
if models.first.id.nil? && column_names.include?(primary_key) && columns_hash[primary_key].type == :uuid
|
@@ -520,12 +535,19 @@ class ActiveRecord::Base
|
|
520
535
|
serialized_attributes
|
521
536
|
end
|
522
537
|
|
523
|
-
array_of_attributes =
|
538
|
+
array_of_attributes = []
|
539
|
+
|
540
|
+
models.each do |model|
|
524
541
|
if supports_setting_primary_key_of_imported_objects?
|
525
542
|
load_association_ids(model)
|
526
543
|
end
|
527
544
|
|
528
|
-
|
545
|
+
if is_validating && !validator.valid_model?(model)
|
546
|
+
raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
|
547
|
+
next
|
548
|
+
end
|
549
|
+
|
550
|
+
array_of_attributes << column_names.map do |name|
|
529
551
|
if stored_attrs.key?(name.to_sym) ||
|
530
552
|
serialized_attrs.key?(name) ||
|
531
553
|
default_values.key?(name.to_s)
|
@@ -597,17 +619,27 @@ class ActiveRecord::Base
|
|
597
619
|
end
|
598
620
|
|
599
621
|
return_obj = if is_validating
|
600
|
-
|
601
|
-
|
602
|
-
models.
|
603
|
-
|
622
|
+
import_with_validations( column_names, array_of_attributes, options ) do |failed_instances|
|
623
|
+
if models
|
624
|
+
models.each { |m| failed_instances << m if m.errors.any? }
|
625
|
+
else
|
626
|
+
# create instances for each of our column/value sets
|
627
|
+
arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes )
|
628
|
+
|
629
|
+
# keep track of the instance and the position it is currently at. if this fails
|
630
|
+
# validation we'll use the index to remove it from the array_of_attributes
|
631
|
+
arr.each_with_index do |hsh, i|
|
632
|
+
model = new
|
633
|
+
hsh.each_pair { |k, v| model[k] = v }
|
634
|
+
next if validator.valid_model?(model)
|
604
635
|
raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
|
605
636
|
array_of_attributes[i] = nil
|
606
|
-
|
637
|
+
failure = model.dup
|
638
|
+
failure.errors.send(:initialize_dup, model.errors)
|
639
|
+
failed_instances << failure
|
607
640
|
end
|
641
|
+
array_of_attributes.compact!
|
608
642
|
end
|
609
|
-
else
|
610
|
-
import_with_validations( column_names, array_of_attributes, options )
|
611
643
|
end
|
612
644
|
else
|
613
645
|
import_without_validations_or_callbacks( column_names, array_of_attributes, options )
|
@@ -640,29 +672,7 @@ class ActiveRecord::Base
|
|
640
672
|
def import_with_validations( column_names, array_of_attributes, options = {} )
|
641
673
|
failed_instances = []
|
642
674
|
|
643
|
-
|
644
|
-
|
645
|
-
if block_given?
|
646
|
-
yield validator, failed_instances
|
647
|
-
else
|
648
|
-
# create instances for each of our column/value sets
|
649
|
-
arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes )
|
650
|
-
|
651
|
-
# keep track of the instance and the position it is currently at. if this fails
|
652
|
-
# validation we'll use the index to remove it from the array_of_attributes
|
653
|
-
arr.each_with_index do |hsh, i|
|
654
|
-
model = new
|
655
|
-
hsh.each_pair { |k, v| model[k] = v }
|
656
|
-
next if validator.valid_model? model
|
657
|
-
raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
|
658
|
-
array_of_attributes[i] = nil
|
659
|
-
failure = model.dup
|
660
|
-
failure.errors.send(:initialize_dup, model.errors)
|
661
|
-
failed_instances << failure
|
662
|
-
end
|
663
|
-
end
|
664
|
-
|
665
|
-
array_of_attributes.compact!
|
675
|
+
yield failed_instances if block_given?
|
666
676
|
|
667
677
|
result = if options[:all_or_none] && failed_instances.any?
|
668
678
|
ActiveRecord::Import::Result.new([], 0, [], [])
|
@@ -945,7 +955,7 @@ Hash key mismatch.
|
|
945
955
|
|
946
956
|
When importing an array of hashes with provided columns_names, each hash must contain keys for all column_names.
|
947
957
|
|
948
|
-
Required keys: #{
|
958
|
+
Required keys: #{required_keys}
|
949
959
|
Missing keys: #{missing_keys}
|
950
960
|
|
951
961
|
Hash: #{hash}
|
@@ -958,7 +968,7 @@ When importing an array of hashes, all hashes must have the same keys.
|
|
958
968
|
If you have records that are missing some values, we recommend you either set default values
|
959
969
|
for the missing keys or group these records into batches by key set before importing.
|
960
970
|
|
961
|
-
Required keys: #{
|
971
|
+
Required keys: #{required_keys}
|
962
972
|
Extra keys: #{extra_keys}
|
963
973
|
Missing keys: #{missing_keys}
|
964
974
|
|
@@ -0,0 +1 @@
|
|
1
|
+
ENV["ARE_DB"] = "postgis"
|
@@ -0,0 +1,8 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
|
2
|
+
require File.expand_path(File.dirname(__FILE__) + '/../support/postgresql/import_examples')
|
3
|
+
|
4
|
+
should_support_postgresql_import_functionality
|
5
|
+
|
6
|
+
if ActiveRecord::Base.connection.supports_on_duplicate_key_update?
|
7
|
+
should_support_postgresql_upsert_functionality
|
8
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
ActiveRecord::Schema.define do
|
2
|
+
create_table :books, force: :cascade do |t|
|
3
|
+
t.string :title, null: false
|
4
|
+
t.virtual :upper_title, type: :string, as: "upper(`title`)" if t.respond_to?(:virtual)
|
5
|
+
t.string :publisher, null: false, default: 'Default Publisher'
|
6
|
+
t.string :author_name, null: false
|
7
|
+
t.datetime :created_at
|
8
|
+
t.datetime :created_on
|
9
|
+
t.datetime :updated_at
|
10
|
+
t.datetime :updated_on
|
11
|
+
t.date :publish_date
|
12
|
+
t.integer :topic_id
|
13
|
+
t.integer :tag_id
|
14
|
+
t.integer :publisher_id
|
15
|
+
t.boolean :for_sale, default: true
|
16
|
+
t.integer :status, default: 0
|
17
|
+
t.string :type
|
18
|
+
end
|
19
|
+
end
|
@@ -82,5 +82,17 @@ def should_support_mysql_import_functionality
|
|
82
82
|
assert_equal "Chad Fowler", topics.last.author_name, "wrong author!"
|
83
83
|
end
|
84
84
|
end
|
85
|
+
|
86
|
+
if ENV['AR_VERSION'].to_f >= 5.1
|
87
|
+
context "with virtual columns" do
|
88
|
+
let(:books) { [Book.new(author_name: "foo", title: "bar")] }
|
89
|
+
|
90
|
+
it "ignores virtual columns and creates record" do
|
91
|
+
assert_difference "Book.count", +1 do
|
92
|
+
Book.import books
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
85
97
|
end
|
86
98
|
end
|
data/test/test_helper.rb
CHANGED
@@ -33,6 +33,14 @@ rescue LoadError
|
|
33
33
|
ENV["SKIP_COMPOSITE_PK"] = "true"
|
34
34
|
end
|
35
35
|
|
36
|
+
# Support MySQL 5.7
|
37
|
+
if ActiveSupport::VERSION::STRING < "4.1"
|
38
|
+
require "active_record/connection_adapters/mysql2_adapter"
|
39
|
+
class ActiveRecord::ConnectionAdapters::Mysql2Adapter
|
40
|
+
NATIVE_DATABASE_TYPES[:primary_key] = "int(11) auto_increment PRIMARY KEY"
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
36
44
|
require "ruby-debug" if RUBY_VERSION.to_f < 1.9
|
37
45
|
|
38
46
|
adapter = ENV["ARE_DB"] || "sqlite3"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: activerecord-import
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.25.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Zach Dennis
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-07-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -101,6 +101,7 @@ files:
|
|
101
101
|
- test/adapters/jdbcmysql.rb
|
102
102
|
- test/adapters/jdbcpostgresql.rb
|
103
103
|
- test/adapters/jdbcsqlite3.rb
|
104
|
+
- test/adapters/makara_postgis.rb
|
104
105
|
- test/adapters/mysql2.rb
|
105
106
|
- test/adapters/mysql2_makara.rb
|
106
107
|
- test/adapters/mysql2spatial.rb
|
@@ -115,6 +116,7 @@ files:
|
|
115
116
|
- test/jdbcmysql/import_test.rb
|
116
117
|
- test/jdbcpostgresql/import_test.rb
|
117
118
|
- test/jdbcsqlite3/import_test.rb
|
119
|
+
- test/makara_postgis/import_test.rb
|
118
120
|
- test/models/account.rb
|
119
121
|
- test/models/alarm.rb
|
120
122
|
- test/models/bike_maker.rb
|
@@ -141,7 +143,7 @@ files:
|
|
141
143
|
- test/postgresql/import_test.rb
|
142
144
|
- test/schema/generic_schema.rb
|
143
145
|
- test/schema/jdbcpostgresql_schema.rb
|
144
|
-
- test/schema/
|
146
|
+
- test/schema/mysql2_schema.rb
|
145
147
|
- test/schema/postgis_schema.rb
|
146
148
|
- test/schema/postgresql_schema.rb
|
147
149
|
- test/schema/version.rb
|
@@ -189,6 +191,7 @@ test_files:
|
|
189
191
|
- test/adapters/jdbcmysql.rb
|
190
192
|
- test/adapters/jdbcpostgresql.rb
|
191
193
|
- test/adapters/jdbcsqlite3.rb
|
194
|
+
- test/adapters/makara_postgis.rb
|
192
195
|
- test/adapters/mysql2.rb
|
193
196
|
- test/adapters/mysql2_makara.rb
|
194
197
|
- test/adapters/mysql2spatial.rb
|
@@ -203,6 +206,7 @@ test_files:
|
|
203
206
|
- test/jdbcmysql/import_test.rb
|
204
207
|
- test/jdbcpostgresql/import_test.rb
|
205
208
|
- test/jdbcsqlite3/import_test.rb
|
209
|
+
- test/makara_postgis/import_test.rb
|
206
210
|
- test/models/account.rb
|
207
211
|
- test/models/alarm.rb
|
208
212
|
- test/models/bike_maker.rb
|
@@ -229,7 +233,7 @@ test_files:
|
|
229
233
|
- test/postgresql/import_test.rb
|
230
234
|
- test/schema/generic_schema.rb
|
231
235
|
- test/schema/jdbcpostgresql_schema.rb
|
232
|
-
- test/schema/
|
236
|
+
- test/schema/mysql2_schema.rb
|
233
237
|
- test/schema/postgis_schema.rb
|
234
238
|
- test/schema/postgresql_schema.rb
|
235
239
|
- test/schema/version.rb
|
data/test/schema/mysql_schema.rb
DELETED
@@ -1,16 +0,0 @@
|
|
1
|
-
ActiveRecord::Schema.define do
|
2
|
-
create_table :books, options: 'ENGINE=MyISAM', force: true do |t|
|
3
|
-
t.column :title, :string, null: false
|
4
|
-
t.column :publisher, :string, null: false, default: 'Default Publisher'
|
5
|
-
t.column :author_name, :string, null: false
|
6
|
-
t.column :created_at, :datetime
|
7
|
-
t.column :created_on, :datetime
|
8
|
-
t.column :updated_at, :datetime
|
9
|
-
t.column :updated_on, :datetime
|
10
|
-
t.column :publish_date, :date
|
11
|
-
t.column :topic_id, :integer
|
12
|
-
t.column :for_sale, :boolean, default: true
|
13
|
-
t.column :status, :integer
|
14
|
-
end
|
15
|
-
execute "ALTER TABLE books ADD FULLTEXT( `title`, `publisher`, `author_name` )"
|
16
|
-
end
|