activerecord-import 0.27.0 → 1.4.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.github/workflows/test.yaml +107 -0
- data/.gitignore +1 -0
- data/.rubocop.yml +74 -8
- data/Brewfile +3 -1
- data/CHANGELOG.md +175 -2
- data/Gemfile +13 -9
- data/LICENSE +21 -56
- data/README.markdown +525 -21
- data/Rakefile +2 -0
- data/activerecord-import.gemspec +6 -5
- data/benchmarks/benchmark.rb +7 -1
- data/benchmarks/lib/base.rb +2 -0
- data/benchmarks/lib/cli_parser.rb +3 -1
- data/benchmarks/lib/float.rb +2 -0
- data/benchmarks/lib/mysql2_benchmark.rb +2 -0
- data/benchmarks/lib/output_to_csv.rb +2 -0
- data/benchmarks/lib/output_to_html.rb +4 -2
- data/benchmarks/models/test_innodb.rb +2 -0
- data/benchmarks/models/test_memory.rb +2 -0
- data/benchmarks/models/test_myisam.rb +2 -0
- data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb} +2 -0
- data/gemfiles/4.2.gemfile +2 -0
- data/gemfiles/5.0.gemfile +2 -0
- data/gemfiles/5.1.gemfile +2 -0
- data/gemfiles/5.2.gemfile +2 -0
- data/gemfiles/6.0.gemfile +4 -0
- data/gemfiles/6.1.gemfile +4 -0
- data/gemfiles/7.0.gemfile +4 -0
- data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb +6 -4
- data/lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/postgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/abstract_adapter.rb +9 -1
- data/lib/activerecord-import/adapters/em_mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql_adapter.rb +10 -11
- data/lib/activerecord-import/adapters/postgresql_adapter.rb +49 -38
- data/lib/activerecord-import/adapters/sqlite3_adapter.rb +21 -25
- data/lib/activerecord-import/base.rb +11 -2
- data/lib/activerecord-import/import.rb +180 -78
- data/lib/activerecord-import/mysql2.rb +2 -0
- data/lib/activerecord-import/postgresql.rb +2 -0
- data/lib/activerecord-import/sqlite3.rb +2 -0
- data/lib/activerecord-import/synchronize.rb +4 -2
- data/lib/activerecord-import/value_sets_parser.rb +4 -0
- data/lib/activerecord-import/version.rb +3 -1
- data/lib/activerecord-import.rb +3 -1
- data/test/adapters/jdbcmysql.rb +2 -0
- data/test/adapters/jdbcpostgresql.rb +2 -0
- data/test/adapters/jdbcsqlite3.rb +2 -0
- data/test/adapters/makara_postgis.rb +2 -0
- data/test/adapters/mysql2.rb +2 -0
- data/test/adapters/mysql2_makara.rb +2 -0
- data/test/adapters/mysql2spatial.rb +2 -0
- data/test/adapters/postgis.rb +2 -0
- data/test/adapters/postgresql.rb +2 -0
- data/test/adapters/postgresql_makara.rb +2 -0
- data/test/adapters/seamless_database_pool.rb +2 -0
- data/test/adapters/spatialite.rb +2 -0
- data/test/adapters/sqlite3.rb +2 -0
- data/test/{travis → github}/database.yml +3 -1
- data/test/import_test.rb +138 -4
- data/test/jdbcmysql/import_test.rb +2 -0
- data/test/jdbcpostgresql/import_test.rb +2 -0
- data/test/jdbcsqlite3/import_test.rb +2 -0
- data/test/makara_postgis/import_test.rb +2 -0
- data/test/models/account.rb +2 -0
- data/test/models/alarm.rb +2 -0
- data/test/models/animal.rb +8 -0
- data/test/models/bike_maker.rb +2 -0
- data/test/models/book.rb +2 -0
- data/test/models/car.rb +2 -0
- data/test/models/card.rb +5 -0
- data/test/models/chapter.rb +2 -0
- data/test/models/customer.rb +8 -0
- data/test/models/deck.rb +8 -0
- data/test/models/dictionary.rb +2 -0
- data/test/models/discount.rb +2 -0
- data/test/models/end_note.rb +2 -0
- data/test/models/group.rb +2 -0
- data/test/models/order.rb +8 -0
- data/test/models/playing_card.rb +4 -0
- data/test/models/promotion.rb +2 -0
- data/test/models/question.rb +2 -0
- data/test/models/rule.rb +2 -0
- data/test/models/tag.rb +3 -0
- data/test/models/tag_alias.rb +5 -0
- data/test/models/topic.rb +2 -0
- data/test/models/user.rb +2 -0
- data/test/models/user_token.rb +3 -0
- data/test/models/vendor.rb +2 -0
- data/test/models/widget.rb +2 -0
- data/test/mysql2/import_test.rb +2 -0
- data/test/mysql2_makara/import_test.rb +2 -0
- data/test/mysqlspatial2/import_test.rb +2 -0
- data/test/postgis/import_test.rb +2 -0
- data/test/postgresql/import_test.rb +2 -0
- data/test/schema/generic_schema.rb +33 -0
- data/test/schema/jdbcpostgresql_schema.rb +2 -0
- data/test/schema/mysql2_schema.rb +2 -0
- data/test/schema/postgis_schema.rb +2 -0
- data/test/schema/postgresql_schema.rb +18 -0
- data/test/schema/sqlite3_schema.rb +2 -0
- data/test/schema/version.rb +2 -0
- data/test/sqlite3/import_test.rb +2 -0
- data/test/support/active_support/test_case_extensions.rb +2 -0
- data/test/support/assertions.rb +2 -0
- data/test/support/factories.rb +2 -0
- data/test/support/generate.rb +4 -2
- data/test/support/mysql/import_examples.rb +2 -1
- data/test/support/postgresql/import_examples.rb +115 -2
- data/test/support/shared_examples/on_duplicate_key_ignore.rb +2 -0
- data/test/support/shared_examples/on_duplicate_key_update.rb +32 -0
- data/test/support/shared_examples/recursive_import.rb +61 -1
- data/test/support/sqlite3/import_examples.rb +4 -16
- data/test/synchronize_test.rb +2 -0
- data/test/test_helper.rb +27 -2
- data/test/value_sets_bytes_parser_test.rb +2 -0
- data/test/value_sets_records_parser_test.rb +2 -0
- metadata +29 -16
- data/.travis.yml +0 -71
- data/gemfiles/3.2.gemfile +0 -2
- data/gemfiles/4.0.gemfile +0 -2
- data/gemfiles/4.1.gemfile +0 -2
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "ostruct"
|
2
4
|
|
3
5
|
module ActiveRecord::Import::ConnectionAdapters; end
|
@@ -24,33 +26,66 @@ module ActiveRecord::Import #:nodoc:
|
|
24
26
|
end
|
25
27
|
|
26
28
|
class Validator
|
27
|
-
def initialize(options = {})
|
29
|
+
def initialize(klass, options = {})
|
28
30
|
@options = options
|
31
|
+
@validator_class = klass
|
32
|
+
init_validations(klass)
|
33
|
+
end
|
34
|
+
|
35
|
+
def init_validations(klass)
|
36
|
+
@validate_callbacks = klass._validate_callbacks.dup
|
37
|
+
|
38
|
+
@validate_callbacks.each_with_index do |callback, i|
|
39
|
+
filter = callback.respond_to?(:raw_filter) ? callback.raw_filter : callback.filter
|
40
|
+
next unless filter.class.name =~ /Validations::PresenceValidator/ ||
|
41
|
+
(!@options[:validate_uniqueness] &&
|
42
|
+
filter.is_a?(ActiveRecord::Validations::UniquenessValidator))
|
43
|
+
|
44
|
+
callback = callback.dup
|
45
|
+
filter = filter.dup
|
46
|
+
attrs = filter.instance_variable_get(:@attributes).dup
|
47
|
+
|
48
|
+
if filter.is_a?(ActiveRecord::Validations::UniquenessValidator)
|
49
|
+
attrs = []
|
50
|
+
else
|
51
|
+
associations = klass.reflect_on_all_associations(:belongs_to)
|
52
|
+
associations.each do |assoc|
|
53
|
+
if (index = attrs.index(assoc.name))
|
54
|
+
key = assoc.foreign_key.is_a?(Array) ? assoc.foreign_key.map(&:to_sym) : assoc.foreign_key.to_sym
|
55
|
+
attrs[index] = key unless attrs.include?(key)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
filter.instance_variable_set(:@attributes, attrs.flatten)
|
61
|
+
|
62
|
+
if @validate_callbacks.respond_to?(:chain, true)
|
63
|
+
@validate_callbacks.send(:chain).tap do |chain|
|
64
|
+
callback.instance_variable_set(:@filter, filter)
|
65
|
+
chain[i] = callback
|
66
|
+
end
|
67
|
+
else
|
68
|
+
callback.raw_filter = filter
|
69
|
+
callback.filter = callback.send(:_compile_filter, filter)
|
70
|
+
@validate_callbacks[i] = callback
|
71
|
+
end
|
72
|
+
end
|
29
73
|
end
|
30
74
|
|
31
75
|
def valid_model?(model)
|
76
|
+
init_validations(model.class) unless model.class == @validator_class
|
77
|
+
|
32
78
|
validation_context = @options[:validate_with_context]
|
33
79
|
validation_context ||= (model.new_record? ? :create : :update)
|
34
|
-
|
35
80
|
current_context = model.send(:validation_context)
|
81
|
+
|
36
82
|
begin
|
37
83
|
model.send(:validation_context=, validation_context)
|
38
84
|
model.errors.clear
|
39
85
|
|
40
|
-
validate_callbacks = model._validate_callbacks.dup
|
41
|
-
associations = model.class.reflect_on_all_associations(:belongs_to).map(&:name)
|
42
|
-
|
43
|
-
model._validate_callbacks.each do |callback|
|
44
|
-
filter = callback.raw_filter
|
45
|
-
if (!@options[:validate_uniqueness] && filter.is_a?(ActiveRecord::Validations::UniquenessValidator)) ||
|
46
|
-
(defined?(ActiveRecord::Validations::PresenceValidator) && filter.is_a?(ActiveRecord::Validations::PresenceValidator) && associations.include?(filter.attributes.first))
|
47
|
-
validate_callbacks.delete(callback)
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
86
|
model.run_callbacks(:validation) do
|
52
87
|
if defined?(ActiveSupport::Callbacks::Filters::Environment) # ActiveRecord >= 4.1
|
53
|
-
runner = validate_callbacks.compile
|
88
|
+
runner = @validate_callbacks.compile
|
54
89
|
env = ActiveSupport::Callbacks::Filters::Environment.new(model, false, nil)
|
55
90
|
if runner.respond_to?(:call) # ActiveRecord < 5.1
|
56
91
|
runner.call(env)
|
@@ -69,10 +104,10 @@ module ActiveRecord::Import #:nodoc:
|
|
69
104
|
runner.invoke_before(env)
|
70
105
|
runner.invoke_after(env)
|
71
106
|
end
|
72
|
-
elsif validate_callbacks.method(:compile).arity == 0 # ActiveRecord = 4.0
|
73
|
-
model.instance_eval validate_callbacks.compile
|
107
|
+
elsif @validate_callbacks.method(:compile).arity == 0 # ActiveRecord = 4.0
|
108
|
+
model.instance_eval @validate_callbacks.compile
|
74
109
|
else # ActiveRecord 3.x
|
75
|
-
model.instance_eval validate_callbacks.compile(nil, model)
|
110
|
+
model.instance_eval @validate_callbacks.compile(nil, model)
|
76
111
|
end
|
77
112
|
end
|
78
113
|
|
@@ -212,16 +247,17 @@ class ActiveRecord::Associations::CollectionAssociation
|
|
212
247
|
alias import bulk_import unless respond_to? :import
|
213
248
|
end
|
214
249
|
|
250
|
+
module ActiveRecord::Import::Connection
|
251
|
+
def establish_connection(args = nil)
|
252
|
+
conn = super(args)
|
253
|
+
ActiveRecord::Import.load_from_connection_pool connection_pool
|
254
|
+
conn
|
255
|
+
end
|
256
|
+
end
|
257
|
+
|
215
258
|
class ActiveRecord::Base
|
216
259
|
class << self
|
217
|
-
|
218
|
-
conn = establish_connection_without_activerecord_import(*args)
|
219
|
-
ActiveRecord::Import.load_from_connection_pool connection_pool
|
220
|
-
conn
|
221
|
-
end
|
222
|
-
|
223
|
-
alias establish_connection_without_activerecord_import establish_connection
|
224
|
-
alias establish_connection establish_connection_with_activerecord_import
|
260
|
+
prepend ActiveRecord::Import::Connection
|
225
261
|
|
226
262
|
# Returns true if the current database connection adapter
|
227
263
|
# supports import functionality, otherwise returns false.
|
@@ -498,7 +534,7 @@ class ActiveRecord::Base
|
|
498
534
|
import_helper(*args)
|
499
535
|
end
|
500
536
|
end
|
501
|
-
alias import bulk_import unless respond_to? :import
|
537
|
+
alias import bulk_import unless ActiveRecord::Base.respond_to? :import
|
502
538
|
|
503
539
|
# Imports a collection of values if all values are valid. Import fails at the
|
504
540
|
# first encountered validation error and raises ActiveRecord::RecordInvalid
|
@@ -510,17 +546,17 @@ class ActiveRecord::Base
|
|
510
546
|
|
511
547
|
bulk_import(*args, options)
|
512
548
|
end
|
513
|
-
alias import! bulk_import! unless respond_to? :import!
|
549
|
+
alias import! bulk_import! unless ActiveRecord::Base.respond_to? :import!
|
514
550
|
|
515
551
|
def import_helper( *args )
|
516
|
-
options = { validate: true, timestamps: true }
|
552
|
+
options = { model: self, validate: true, timestamps: true, track_validation_failures: false }
|
517
553
|
options.merge!( args.pop ) if args.last.is_a? Hash
|
518
554
|
# making sure that current model's primary key is used
|
519
555
|
options[:primary_key] = primary_key
|
520
556
|
options[:locking_column] = locking_column if attribute_names.include?(locking_column)
|
521
557
|
|
522
558
|
is_validating = options[:validate_with_context].present? ? true : options[:validate]
|
523
|
-
validator = ActiveRecord::Import::Validator.new(options)
|
559
|
+
validator = ActiveRecord::Import::Validator.new(self, options)
|
524
560
|
|
525
561
|
# assume array of model objects
|
526
562
|
if args.last.is_a?( Array ) && args.last.first.is_a?(ActiveRecord::Base)
|
@@ -544,13 +580,12 @@ class ActiveRecord::Base
|
|
544
580
|
end
|
545
581
|
end
|
546
582
|
|
547
|
-
|
548
|
-
|
549
|
-
|
550
|
-
|
551
|
-
|
552
|
-
|
553
|
-
serialized_attributes
|
583
|
+
update_attrs = if record_timestamps && options[:timestamps]
|
584
|
+
if respond_to?(:timestamp_attributes_for_update, true)
|
585
|
+
send(:timestamp_attributes_for_update).map(&:to_sym)
|
586
|
+
else
|
587
|
+
allocate.send(:timestamp_attributes_for_update_in_model)
|
588
|
+
end
|
554
589
|
end
|
555
590
|
|
556
591
|
array_of_attributes = []
|
@@ -566,12 +601,12 @@ class ActiveRecord::Base
|
|
566
601
|
end
|
567
602
|
|
568
603
|
array_of_attributes << column_names.map do |name|
|
569
|
-
if
|
570
|
-
|
571
|
-
|
572
|
-
|
604
|
+
if model.persisted? &&
|
605
|
+
update_attrs && update_attrs.include?(name.to_sym) &&
|
606
|
+
!model.send("#{name}_changed?")
|
607
|
+
nil
|
573
608
|
else
|
574
|
-
model.
|
609
|
+
model.read_attribute(name.to_s)
|
575
610
|
end
|
576
611
|
end
|
577
612
|
end
|
@@ -598,7 +633,7 @@ class ActiveRecord::Base
|
|
598
633
|
end
|
599
634
|
# supports empty array
|
600
635
|
elsif args.last.is_a?( Array ) && args.last.empty?
|
601
|
-
return ActiveRecord::Import::Result.new([], 0, [])
|
636
|
+
return ActiveRecord::Import::Result.new([], 0, [], [])
|
602
637
|
# supports 2-element array and array
|
603
638
|
elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array )
|
604
639
|
|
@@ -655,7 +690,7 @@ class ActiveRecord::Base
|
|
655
690
|
timestamps = {}
|
656
691
|
|
657
692
|
# record timestamps unless disabled in ActiveRecord::Base
|
658
|
-
if record_timestamps && options
|
693
|
+
if record_timestamps && options[:timestamps]
|
659
694
|
timestamps = add_special_rails_stamps column_names, array_of_attributes, options
|
660
695
|
end
|
661
696
|
|
@@ -670,14 +705,18 @@ class ActiveRecord::Base
|
|
670
705
|
# keep track of the instance and the position it is currently at. if this fails
|
671
706
|
# validation we'll use the index to remove it from the array_of_attributes
|
672
707
|
arr.each_with_index do |hsh, i|
|
673
|
-
|
674
|
-
|
708
|
+
# utilize block initializer syntax to prevent failure when 'mass_assignment_sanitizer = :strict'
|
709
|
+
model = new do |m|
|
710
|
+
hsh.each_pair { |k, v| m[k] = v }
|
711
|
+
end
|
712
|
+
|
675
713
|
next if validator.valid_model?(model)
|
676
714
|
raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
|
715
|
+
|
677
716
|
array_of_attributes[i] = nil
|
678
717
|
failure = model.dup
|
679
718
|
failure.errors.send(:initialize_dup, model.errors)
|
680
|
-
failed_instances << failure
|
719
|
+
failed_instances << (options[:track_validation_failures] ? [i, failure] : failure )
|
681
720
|
end
|
682
721
|
array_of_attributes.compact!
|
683
722
|
end
|
@@ -697,7 +736,10 @@ class ActiveRecord::Base
|
|
697
736
|
set_attributes_and_mark_clean(models, return_obj, timestamps, options)
|
698
737
|
|
699
738
|
# if there are auto-save associations on the models we imported that are new, import them as well
|
700
|
-
|
739
|
+
if options[:recursive]
|
740
|
+
options[:on_duplicate_key_update] = on_duplicate_key_update unless on_duplicate_key_update.nil?
|
741
|
+
import_associations(models, options.dup.merge(validate: false))
|
742
|
+
end
|
701
743
|
end
|
702
744
|
|
703
745
|
return_obj
|
@@ -738,21 +780,22 @@ class ActiveRecord::Base
|
|
738
780
|
unless scope_columns.blank?
|
739
781
|
scope_columns.zip(scope_values).each do |name, value|
|
740
782
|
name_as_sym = name.to_sym
|
741
|
-
next if column_names.include?(name_as_sym)
|
742
|
-
|
743
|
-
is_sti = (name_as_sym == inheritance_column.to_sym && self < base_class)
|
744
|
-
value = Array(value).first if is_sti
|
745
|
-
|
783
|
+
next if column_names.include?(name_as_sym) || name_as_sym == inheritance_column.to_sym
|
746
784
|
column_names << name_as_sym
|
747
785
|
array_of_attributes.each { |attrs| attrs << value }
|
748
786
|
end
|
749
787
|
end
|
750
788
|
|
789
|
+
if finder_needs_type_condition?
|
790
|
+
unless column_names.include?(inheritance_column.to_sym)
|
791
|
+
column_names << inheritance_column.to_sym
|
792
|
+
array_of_attributes.each { |attrs| attrs << sti_name }
|
793
|
+
end
|
794
|
+
end
|
795
|
+
|
751
796
|
columns = column_names.each_with_index.map do |name, i|
|
752
797
|
column = columns_hash[name.to_s]
|
753
|
-
|
754
798
|
raise ActiveRecord::Import::MissingColumnError.new(name.to_s, i) if column.nil?
|
755
|
-
|
756
799
|
column
|
757
800
|
end
|
758
801
|
|
@@ -768,17 +811,29 @@ class ActiveRecord::Base
|
|
768
811
|
if supports_import?
|
769
812
|
# generate the sql
|
770
813
|
post_sql_statements = connection.post_sql_statements( quoted_table_name, options )
|
814
|
+
import_size = values_sql.size
|
815
|
+
|
816
|
+
batch_size = options[:batch_size] || import_size
|
817
|
+
run_proc = options[:batch_size].to_i.positive? && options[:batch_progress].respond_to?( :call )
|
818
|
+
progress_proc = options[:batch_progress]
|
819
|
+
current_batch = 0
|
820
|
+
batches = (import_size / batch_size.to_f).ceil
|
771
821
|
|
772
|
-
batch_size = options[:batch_size] || values_sql.size
|
773
822
|
values_sql.each_slice(batch_size) do |batch_values|
|
823
|
+
batch_started_at = Time.now.to_i
|
824
|
+
|
774
825
|
# perform the inserts
|
775
826
|
result = connection.insert_many( [insert_sql, post_sql_statements].flatten,
|
776
827
|
batch_values,
|
777
828
|
options,
|
778
|
-
"#{model_name} Create Many
|
829
|
+
"#{model_name} Create Many" )
|
830
|
+
|
779
831
|
number_inserted += result.num_inserts
|
780
832
|
ids += result.ids
|
781
833
|
results += result.results
|
834
|
+
current_batch += 1
|
835
|
+
|
836
|
+
progress_proc.call(import_size, batches, current_batch, Time.now.to_i - batch_started_at) if run_proc
|
782
837
|
end
|
783
838
|
else
|
784
839
|
transaction(requires_new: true) do
|
@@ -804,29 +859,56 @@ class ActiveRecord::Base
|
|
804
859
|
model.id = id
|
805
860
|
|
806
861
|
timestamps.each do |attr, value|
|
807
|
-
model.send(attr + "=", value)
|
862
|
+
model.send(attr + "=", value) if model.send(attr).nil?
|
808
863
|
end
|
809
864
|
end
|
810
865
|
end
|
811
866
|
|
812
|
-
|
813
|
-
|
814
|
-
|
815
|
-
|
867
|
+
deserialize_value = lambda do |column, value|
|
868
|
+
column = columns_hash[column]
|
869
|
+
return value unless column
|
870
|
+
if respond_to?(:type_caster)
|
871
|
+
type = type_for_attribute(column.name)
|
872
|
+
type.deserialize(value)
|
873
|
+
elsif column.respond_to?(:type_cast_from_database)
|
874
|
+
column.type_cast_from_database(value)
|
875
|
+
else
|
876
|
+
value
|
877
|
+
end
|
878
|
+
end
|
879
|
+
|
880
|
+
set_value = lambda do |model, column, value|
|
881
|
+
val = deserialize_value.call(column, value)
|
882
|
+
if model.attribute_names.include?(column)
|
883
|
+
model.send("#{column}=", val)
|
884
|
+
else
|
885
|
+
attributes = attributes_builder.build_from_database(model.attributes.merge(column => val))
|
886
|
+
model.instance_variable_set(:@attributes, attributes)
|
887
|
+
end
|
888
|
+
end
|
889
|
+
|
890
|
+
columns = Array(options[:returning_columns])
|
891
|
+
results = Array(import_result.results)
|
892
|
+
if models.size == results.size
|
893
|
+
single_column = columns.first if columns.size == 1
|
894
|
+
results.each_with_index do |result, index|
|
816
895
|
model = models[index]
|
817
896
|
|
818
897
|
if single_column
|
819
|
-
|
898
|
+
set_value.call(model, single_column, result)
|
820
899
|
else
|
821
900
|
columns.each_with_index do |column, col_index|
|
822
|
-
|
901
|
+
set_value.call(model, column, result[col_index])
|
823
902
|
end
|
824
903
|
end
|
825
904
|
end
|
826
905
|
end
|
827
906
|
|
828
907
|
models.each do |model|
|
829
|
-
if model.respond_to?(:
|
908
|
+
if model.respond_to?(:changes_applied) # Rails 4.1.8 and higher
|
909
|
+
model.changes_internally_applied if model.respond_to?(:changes_internally_applied) # legacy behavior for Rails 5.1
|
910
|
+
model.changes_applied
|
911
|
+
elsif model.respond_to?(:clear_changes_information) # Rails 4.0 and higher
|
830
912
|
model.clear_changes_information
|
831
913
|
else # Rails 3.2
|
832
914
|
model.instance_variable_get(:@changed_attributes).clear
|
@@ -837,16 +919,22 @@ class ActiveRecord::Base
|
|
837
919
|
|
838
920
|
# Sync belongs_to association ids with foreign key field
|
839
921
|
def load_association_ids(model)
|
922
|
+
changed_columns = model.changed
|
840
923
|
association_reflections = model.class.reflect_on_all_associations(:belongs_to)
|
841
924
|
association_reflections.each do |association_reflection|
|
842
|
-
column_name = association_reflection.foreign_key
|
843
925
|
next if association_reflection.options[:polymorphic]
|
844
|
-
association = model.association(association_reflection.name)
|
845
|
-
association = association.target
|
846
|
-
next if association.blank? || model.public_send(column_name).present?
|
847
926
|
|
848
|
-
|
849
|
-
|
927
|
+
column_names = Array(association_reflection.foreign_key).map(&:to_s)
|
928
|
+
column_names.each_with_index do |column_name, column_index|
|
929
|
+
next if changed_columns.include?(column_name)
|
930
|
+
|
931
|
+
association = model.association(association_reflection.name)
|
932
|
+
association = association.target
|
933
|
+
next if association.blank? || model.public_send(column_name).present?
|
934
|
+
|
935
|
+
association_primary_key = Array(association_reflection.association_primary_key)[column_index]
|
936
|
+
model.public_send("#{column_name}=", association.send(association_primary_key))
|
937
|
+
end
|
850
938
|
end
|
851
939
|
end
|
852
940
|
|
@@ -859,8 +947,10 @@ class ActiveRecord::Base
|
|
859
947
|
associated_objects_by_class = {}
|
860
948
|
models.each { |model| find_associated_objects_for_import(associated_objects_by_class, model) }
|
861
949
|
|
862
|
-
# :on_duplicate_key_update
|
863
|
-
options.delete(:on_duplicate_key_update)
|
950
|
+
# :on_duplicate_key_update only supported for all fields
|
951
|
+
options.delete(:on_duplicate_key_update) unless options[:on_duplicate_key_update] == :all
|
952
|
+
# :returning not supported for associations
|
953
|
+
options.delete(:returning)
|
864
954
|
|
865
955
|
associated_objects_by_class.each_value do |associations|
|
866
956
|
associations.each_value do |associated_records|
|
@@ -891,8 +981,13 @@ class ActiveRecord::Base
|
|
891
981
|
changed_objects.each do |child|
|
892
982
|
child.public_send("#{association_reflection.foreign_key}=", model.id)
|
893
983
|
# For polymorphic associations
|
984
|
+
association_name = if model.class.respond_to?(:polymorphic_name)
|
985
|
+
model.class.polymorphic_name
|
986
|
+
else
|
987
|
+
model.class.base_class
|
988
|
+
end
|
894
989
|
association_reflection.type.try do |type|
|
895
|
-
child.public_send("#{type}=",
|
990
|
+
child.public_send("#{type}=", association_name)
|
896
991
|
end
|
897
992
|
end
|
898
993
|
associated_objects_by_class[model.class.name][association_reflection.name].concat changed_objects
|
@@ -919,7 +1014,7 @@ class ActiveRecord::Base
|
|
919
1014
|
elsif column
|
920
1015
|
if respond_to?(:type_caster) # Rails 5.0 and higher
|
921
1016
|
type = type_for_attribute(column.name)
|
922
|
-
val = type.type == :boolean ? type.cast(val) : type.serialize(val)
|
1017
|
+
val = !type.respond_to?(:subtype) && type.type == :boolean ? type.cast(val) : type.serialize(val)
|
923
1018
|
connection_memo.quote(val)
|
924
1019
|
elsif column.respond_to?(:type_cast_from_user) # Rails 4.2
|
925
1020
|
connection_memo.quote(column.type_cast_from_user(val), column)
|
@@ -928,9 +1023,11 @@ class ActiveRecord::Base
|
|
928
1023
|
val = serialized_attributes[column.name].dump(val)
|
929
1024
|
end
|
930
1025
|
# Fixes #443 to support binary (i.e. bytea) columns on PG
|
931
|
-
val = column.type_cast(val) unless column.type.to_sym == :binary
|
1026
|
+
val = column.type_cast(val) unless column.type && column.type.to_sym == :binary
|
932
1027
|
connection_memo.quote(val, column)
|
933
1028
|
end
|
1029
|
+
else
|
1030
|
+
raise ArgumentError, "Number of values (#{arr.length}) exceeds number of columns (#{columns.length})"
|
934
1031
|
end
|
935
1032
|
end
|
936
1033
|
"(#{my_values.join(',')})"
|
@@ -945,13 +1042,18 @@ class ActiveRecord::Base
|
|
945
1042
|
timestamp_columns[:create] = timestamp_attributes_for_create_in_model
|
946
1043
|
timestamp_columns[:update] = timestamp_attributes_for_update_in_model
|
947
1044
|
else
|
948
|
-
instance =
|
1045
|
+
instance = allocate
|
949
1046
|
timestamp_columns[:create] = instance.send(:timestamp_attributes_for_create_in_model)
|
950
1047
|
timestamp_columns[:update] = instance.send(:timestamp_attributes_for_update_in_model)
|
951
1048
|
end
|
952
1049
|
|
953
1050
|
# use tz as set in ActiveRecord::Base
|
954
|
-
|
1051
|
+
default_timezone = if ActiveRecord.respond_to?(:default_timezone)
|
1052
|
+
ActiveRecord.default_timezone
|
1053
|
+
else
|
1054
|
+
ActiveRecord::Base.default_timezone
|
1055
|
+
end
|
1056
|
+
timestamp = default_timezone == :utc ? Time.now.utc : Time.now
|
955
1057
|
|
956
1058
|
[:create, :update].each do |action|
|
957
1059
|
timestamp_columns[action].each do |column|
|
@@ -961,7 +1063,7 @@ class ActiveRecord::Base
|
|
961
1063
|
index = column_names.index(column) || column_names.index(column.to_sym)
|
962
1064
|
if index
|
963
1065
|
# replace every instance of the array of attributes with our value
|
964
|
-
array_of_attributes.each { |arr| arr[index] = timestamp if arr[index].nil?
|
1066
|
+
array_of_attributes.each { |arr| arr[index] = timestamp if arr[index].nil? }
|
965
1067
|
else
|
966
1068
|
column_names << column
|
967
1069
|
array_of_attributes.each { |arr| arr << timestamp }
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module ActiveRecord # :nodoc:
|
2
4
|
class Base # :nodoc:
|
3
5
|
# Synchronizes the passed in ActiveRecord instances with data
|
@@ -39,8 +41,8 @@ module ActiveRecord # :nodoc:
|
|
39
41
|
|
40
42
|
next unless matched_instance
|
41
43
|
|
42
|
-
instance.
|
43
|
-
instance.send :
|
44
|
+
instance.instance_variable_set :@association_cache, {}
|
45
|
+
instance.send :clear_aggregation_cache if instance.respond_to?(:clear_aggregation_cache, true)
|
44
46
|
instance.instance_variable_set :@attributes, matched_instance.instance_variable_get(:@attributes)
|
45
47
|
|
46
48
|
if instance.respond_to?(:clear_changes_information)
|
data/lib/activerecord-import.rb
CHANGED
data/test/adapters/jdbcmysql.rb
CHANGED
data/test/adapters/mysql2.rb
CHANGED
data/test/adapters/postgis.rb
CHANGED
data/test/adapters/postgresql.rb
CHANGED
data/test/adapters/spatialite.rb
CHANGED
data/test/adapters/sqlite3.rb
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
common: &common
|
2
2
|
username: root
|
3
|
-
password:
|
3
|
+
password: root
|
4
4
|
encoding: utf8
|
5
|
+
collation: utf8_general_ci
|
5
6
|
host: localhost
|
6
7
|
database: activerecord_import_test
|
7
8
|
|
@@ -37,6 +38,7 @@ oracle:
|
|
37
38
|
postgresql: &postgresql
|
38
39
|
<<: *common
|
39
40
|
username: postgres
|
41
|
+
password: postgres
|
40
42
|
adapter: postgresql
|
41
43
|
min_messages: warning
|
42
44
|
|