activerecord-import 1.0.7 → 1.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/test.yaml +78 -0
- data/CHANGELOG.md +40 -3
- data/Gemfile +4 -1
- data/README.markdown +16 -4
- data/activerecord-import.gemspec +2 -2
- data/gemfiles/6.0.gemfile +1 -0
- data/gemfiles/6.1.gemfile +2 -1
- data/gemfiles/7.0.gemfile +1 -0
- data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb +4 -4
- data/lib/activerecord-import/adapters/postgresql_adapter.rb +1 -3
- data/lib/activerecord-import/adapters/sqlite3_adapter.rb +0 -2
- data/lib/activerecord-import/import.rb +54 -23
- data/lib/activerecord-import/synchronize.rb +1 -1
- data/lib/activerecord-import/value_sets_parser.rb +2 -0
- data/lib/activerecord-import/version.rb +1 -1
- data/test/{travis → github}/database.yml +3 -1
- data/test/import_test.rb +28 -1
- data/test/models/card.rb +3 -0
- data/test/models/customer.rb +6 -0
- data/test/models/deck.rb +6 -0
- data/test/models/order.rb +6 -0
- data/test/models/playing_card.rb +2 -0
- data/test/schema/generic_schema.rb +25 -0
- data/test/support/postgresql/import_examples.rb +24 -0
- data/test/support/shared_examples/recursive_import.rb +21 -1
- data/test/test_helper.rb +10 -1
- metadata +23 -15
- data/.travis.yml +0 -74
- data/gemfiles/3.2.gemfile +0 -2
- data/gemfiles/4.0.gemfile +0 -2
- data/gemfiles/4.1.gemfile +0 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 815280c3b17a8ee84e54defd450a68fa532719dc8760ac001883f3df0156d689
|
4
|
+
data.tar.gz: d34ee8c4c186b172259ea4f3427a75379d08b7ae41ee67128fd870e489d2f78d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 49c6d78cbcdc342db44497ec66f0dfe466b08124dc233996688c9443b0c455328e6d7ebc80d011c8732f7ae5b5468bd3c4f57f4abe2bb20bc26a118183cea402
|
7
|
+
data.tar.gz: 9f602bb4423fb3cc5169727d150df7eef5396954f0576f1ecf8cb168179adc625f5ef5927a4655eb8271e37d845d9e5c6b6a6a597a83cb1c268958b4f2f65375
|
@@ -0,0 +1,78 @@
|
|
1
|
+
name: Test
|
2
|
+
on: [push, pull_request]
|
3
|
+
jobs:
|
4
|
+
test:
|
5
|
+
services:
|
6
|
+
postgres:
|
7
|
+
image: postgis/postgis:10-2.5
|
8
|
+
env:
|
9
|
+
POSTGRES_USER: postgres
|
10
|
+
POSTGRES_PASSWORD: postgres
|
11
|
+
ports:
|
12
|
+
- 5432:5432
|
13
|
+
# Set health checks to wait until postgres has started
|
14
|
+
options: >-
|
15
|
+
--health-cmd pg_isready
|
16
|
+
--health-interval 10s
|
17
|
+
--health-timeout 5s
|
18
|
+
--health-retries 5
|
19
|
+
strategy:
|
20
|
+
fail-fast: false
|
21
|
+
matrix:
|
22
|
+
ruby:
|
23
|
+
- 2.7
|
24
|
+
env:
|
25
|
+
- AR_VERSION: 7.0
|
26
|
+
- AR_VERSION: 6.1
|
27
|
+
- AR_VERSION: 6.0
|
28
|
+
include:
|
29
|
+
- ruby: 2.6
|
30
|
+
env:
|
31
|
+
AR_VERSION: 5.2
|
32
|
+
- ruby: 2.6
|
33
|
+
env:
|
34
|
+
AR_VERSION: 5.1
|
35
|
+
- ruby: 2.4
|
36
|
+
env:
|
37
|
+
AR_VERSION: 5.0
|
38
|
+
- ruby: 2.4
|
39
|
+
env:
|
40
|
+
AR_VERSION: 4.2
|
41
|
+
runs-on: ubuntu-latest
|
42
|
+
env:
|
43
|
+
AR_VERSION: ${{ matrix.env.AR_VERSION }}
|
44
|
+
DB_DATABASE: activerecord_import_test
|
45
|
+
steps:
|
46
|
+
- uses: actions/checkout@v2
|
47
|
+
- uses: ruby/setup-ruby@v1
|
48
|
+
with:
|
49
|
+
ruby-version: ${{ matrix.ruby }}
|
50
|
+
- name: Setup Bundler 1.x for Ruby 2.3
|
51
|
+
if: ${{ matrix.ruby == '2.3' }}
|
52
|
+
run: echo "BUNDLER_VERSION=1.17.3" >> $GITHUB_ENV
|
53
|
+
- name: Set up databases
|
54
|
+
run: |
|
55
|
+
sudo /etc/init.d/mysql start
|
56
|
+
mysql -e 'CREATE DATABASE ${{ env.DB_DATABASE }} CHARACTER SET utf8 COLLATE utf8_general_ci;' -u root -proot
|
57
|
+
psql -h localhost -U postgres -c 'create database ${{ env.DB_DATABASE }};'
|
58
|
+
psql -h localhost -U postgres -d ${{ env.DB_DATABASE }} -c 'create extension if not exists hstore;'
|
59
|
+
psql -h localhost -U postgres -c 'create extension if not exists postgis;'
|
60
|
+
psql -h localhost -U postgres -c 'create extension if not exists "uuid-ossp";'
|
61
|
+
cp test/github/database.yml test/database.yml
|
62
|
+
env:
|
63
|
+
PGPASSWORD: postgres
|
64
|
+
- name: Install dependencies
|
65
|
+
run : AR_VERSION=${{ env.AR_VERSION }} bundle install
|
66
|
+
- name: Run tests
|
67
|
+
run: |
|
68
|
+
bundle exec rake test:mysql2
|
69
|
+
bundle exec rake test:mysql2_makara
|
70
|
+
bundle exec rake test:mysql2spatial
|
71
|
+
bundle exec rake test:postgis
|
72
|
+
bundle exec rake test:postgresql
|
73
|
+
bundle exec rake test:postgresql_makara
|
74
|
+
bundle exec rake test:seamless_database_pool
|
75
|
+
bundle exec rake test:spatialite
|
76
|
+
bundle exec rake test:sqlite3
|
77
|
+
- name: Run Rubocop
|
78
|
+
run: bundle exec rubocop
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,42 @@
|
|
1
|
+
## Changes in 1.3.0
|
2
|
+
|
3
|
+
### Fixes
|
4
|
+
|
5
|
+
* Ensure correct timestamp values are returned for models after insert. Thanks to @kos1kov via \##756.
|
6
|
+
* Restore database_version method to public scope. Thanks to @beauraF via \##753.
|
7
|
+
|
8
|
+
### New Features
|
9
|
+
|
10
|
+
* Add support for ActiveRecord 7.0. Thanks to @nickhammond, @ryanwood, @jkowens via \##749 and \##752.
|
11
|
+
* Add support for compound foreign keys. Thanks to @Uladzimiro via \##750.
|
12
|
+
* Add support for :recursive combined with on_duplicate_key_update: :all. Thanks to @deathwish via \##746.
|
13
|
+
|
14
|
+
## Changes in 1.2.0
|
15
|
+
|
16
|
+
### Fixes
|
17
|
+
|
18
|
+
* Update JDBC MySQL adapter to use mysql2 connection adapter. Thanks to @terencechow via \##744.
|
19
|
+
* Fix importing STI models with ActiveRecord 6. Thanks to @clemens1483 via \##743.
|
20
|
+
* Use polymorphic_name instead of base_class.name for imports. Thanks to @kmhajjar via \##741.
|
21
|
+
* Fix compatibility issue with composite primary keys. Thanks to @dlanileonardo via \##737.
|
22
|
+
* Prevent double validation of associations on recursive import.
|
23
|
+
|
24
|
+
## Changes in 1.1.0
|
25
|
+
|
26
|
+
### New Features
|
27
|
+
|
28
|
+
* Add batch progress reporting. Thanks to @gee-forr via \##729.
|
29
|
+
|
30
|
+
## Changes in 1.0.8
|
31
|
+
|
32
|
+
### Fixes
|
33
|
+
|
34
|
+
* Use correct method for clearing query cache. Thanks to @EtienneDepaulis via \##719.
|
35
|
+
|
1
36
|
## Changes in 1.0.7
|
2
37
|
|
38
|
+
### New Features
|
39
|
+
|
3
40
|
* Use @@max_allowed_packet session variable instead of querying SHOW VARIABLES. Thanks to @diclophis via \#706.
|
4
41
|
* Add option :track_validation_failures. When this is set to true, failed_instances will be an array of arrays, with each inner array having the form [:index_in_dataset, :object_with_errors]. Thanks to @rorymckinley via \#684.
|
5
42
|
|
@@ -13,7 +50,7 @@
|
|
13
50
|
|
14
51
|
* Handle after_initialize callbacks. Thanks to @AhMohsen46 via \#691 and
|
15
52
|
\#692.
|
16
|
-
* Fix regression introduced in 1.0.4.
|
53
|
+
* Fix regression introduced in 1.0.4. Explicitly allow adapters to
|
17
54
|
support on duplicate key update. Thanks to @dsobiera, @jkowens via \#696.
|
18
55
|
|
19
56
|
## Changes in 1.0.5
|
@@ -22,7 +59,7 @@
|
|
22
59
|
|
23
60
|
* Allow serialized attributes to be returned from import. Thanks to @timanovsky, @jkowens via \#660.
|
24
61
|
* Return ActiveRecord::Connection from
|
25
|
-
|
62
|
+
ActiveRecord::Base#establish_connection. Thanks to @reverentF via
|
26
63
|
\#663.
|
27
64
|
* Support PostgreSQL array. Thanks to @ujihisa via \#669.
|
28
65
|
* Skip loading association ids when column changed. Thanks to @Aristat
|
@@ -77,7 +114,7 @@
|
|
77
114
|
* Fix import issue for models with Postgresql json/jsonb fields. Thanks
|
78
115
|
to @stokarenko via \#594.
|
79
116
|
* Fix issue importing models with timestamps that contain timezone
|
80
|
-
information.
|
117
|
+
information. Thanks to @dekaikiwi, @jkowens via \#598.
|
81
118
|
* Ignore :no_returning when using :recursive option. Thanks to @dgollahon, @jkowens
|
82
119
|
via \#599.
|
83
120
|
|
data/Gemfile
CHANGED
@@ -6,8 +6,11 @@ version = ENV['AR_VERSION'].to_f
|
|
6
6
|
|
7
7
|
mysql2_version = '0.3.0'
|
8
8
|
mysql2_version = '0.4.0' if version >= 4.2
|
9
|
+
mysql2_version = '0.5.0' if version >= 6.1
|
9
10
|
sqlite3_version = '1.3.0'
|
10
11
|
sqlite3_version = '1.4.0' if version >= 6.0
|
12
|
+
pg_version = '0.9'
|
13
|
+
pg_version = '1.1' if version >= 6.1
|
11
14
|
|
12
15
|
group :development, :test do
|
13
16
|
gem 'rubocop', '~> 0.40.0'
|
@@ -17,7 +20,7 @@ end
|
|
17
20
|
# Database Adapters
|
18
21
|
platforms :ruby do
|
19
22
|
gem "mysql2", "~> #{mysql2_version}"
|
20
|
-
gem "pg", "~>
|
23
|
+
gem "pg", "~> #{pg_version}"
|
21
24
|
gem "sqlite3", "~> #{sqlite3_version}"
|
22
25
|
gem "seamless_database_pool", "~> 1.0.20"
|
23
26
|
end
|
data/README.markdown
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
# Activerecord-Import
|
1
|
+
# Activerecord-Import ![Build Status](https://github.com/zdennis/activerecord-import/actions/workflows/test.yaml/badge.svg)
|
2
2
|
|
3
3
|
Activerecord-Import is a library for bulk inserting data using ActiveRecord.
|
4
4
|
|
@@ -231,6 +231,18 @@ columns = [ :title ]
|
|
231
231
|
Book.import columns, books, batch_size: 2
|
232
232
|
```
|
233
233
|
|
234
|
+
If your import is particularly large or slow (possibly due to [callbacks](#callbacks)) whilst batch importing, you might want a way to report back on progress. This is supported by passing a callable as the `batch_progress` option. e.g:
|
235
|
+
|
236
|
+
```ruby
|
237
|
+
my_proc = ->(rows_size, num_batches, current_batch_number, batch_duration_in_secs) {
|
238
|
+
# Using the arguments provided to the callable, you can
|
239
|
+
# send an email, post to a websocket,
|
240
|
+
# update slack, alert if import is taking too long, etc.
|
241
|
+
}
|
242
|
+
|
243
|
+
Book.import columns, books, batch_size: 2, batch_progress: my_proc
|
244
|
+
```
|
245
|
+
|
234
246
|
#### Recursive
|
235
247
|
|
236
248
|
NOTE: This only works with PostgreSQL and ActiveRecord objects. This won't work with
|
@@ -254,8 +266,8 @@ Key | Options | Default | Descrip
|
|
254
266
|
------------------------- | --------------------- | ------------------ | -----------
|
255
267
|
:validate | `true`/`false` | `true` | Whether or not to run `ActiveRecord` validations (uniqueness skipped). This option will always be true when using `import!`.
|
256
268
|
:validate_uniqueness | `true`/`false` | `false` | Whether or not to run uniqueness validations, has potential pitfalls, use with caution (requires `>= v0.27.0`).
|
257
|
-
:validate_with_context | `Symbol` |`:create`/`:update` | Allows passing an ActiveModel validation context for each model. Default is `:create` for new records and `:update` for existing ones.
|
258
|
-
:track_validation_failures| `true`/`false` | `false` | When this is set to true, `failed_instances` will be an array of arrays, with each inner array having the form `[:index_in_dataset, :object_with_errors]`
|
269
|
+
:validate_with_context | `Symbol` |`:create`/`:update` | Allows passing an ActiveModel validation context for each model. Default is `:create` for new records and `:update` for existing ones.
|
270
|
+
:track_validation_failures| `true`/`false` | `false` | When this is set to true, `failed_instances` will be an array of arrays, with each inner array having the form `[:index_in_dataset, :object_with_errors]`
|
259
271
|
:on_duplicate_key_ignore | `true`/`false` | `false` | Allows skipping records with duplicate keys. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-ignore) for more details.
|
260
272
|
:ignore | `true`/`false` | `false` | Alias for :on_duplicate_key_ignore.
|
261
273
|
:on_duplicate_key_update | :all, `Array`, `Hash` | N/A | Allows upsert logic to be used. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-update) for more details.
|
@@ -380,7 +392,7 @@ Book.import books, validate_uniqueness: true
|
|
380
392
|
|
381
393
|
### Return Info
|
382
394
|
|
383
|
-
The `import` method returns a `Result` object that responds to `failed_instances` and `num_inserts`. Additionally, for users of Postgres, there will be two arrays `ids` and `results` that can be accessed
|
395
|
+
The `import` method returns a `Result` object that responds to `failed_instances` and `num_inserts`. Additionally, for users of Postgres, there will be two arrays `ids` and `results` that can be accessed.
|
384
396
|
|
385
397
|
```ruby
|
386
398
|
articles = [
|
data/activerecord-import.gemspec
CHANGED
@@ -16,8 +16,8 @@ Gem::Specification.new do |gem|
|
|
16
16
|
gem.require_paths = ["lib"]
|
17
17
|
gem.version = ActiveRecord::Import::VERSION
|
18
18
|
|
19
|
-
gem.required_ruby_version = ">= 2.
|
19
|
+
gem.required_ruby_version = ">= 2.4.0"
|
20
20
|
|
21
|
-
gem.add_runtime_dependency "activerecord", ">=
|
21
|
+
gem.add_runtime_dependency "activerecord", ">= 4.2"
|
22
22
|
gem.add_development_dependency "rake"
|
23
23
|
end
|
data/gemfiles/6.0.gemfile
CHANGED
data/gemfiles/6.1.gemfile
CHANGED
@@ -1 +1,2 @@
|
|
1
|
-
gem 'activerecord', '~> 6.1.0
|
1
|
+
gem 'activerecord', '~> 6.1.0'
|
2
|
+
gem 'composite_primary_keys', '~> 13.0'
|
@@ -0,0 +1 @@
|
|
1
|
+
gem 'activerecord', '~> 7.0.0.alpha2'
|
@@ -1,6 +1,6 @@
|
|
1
|
-
require "active_record/connection_adapters/
|
2
|
-
require "activerecord-import/adapters/
|
1
|
+
require "active_record/connection_adapters/mysql2_adapter"
|
2
|
+
require "activerecord-import/adapters/mysql2_adapter"
|
3
3
|
|
4
|
-
class ActiveRecord::ConnectionAdapters::
|
5
|
-
include ActiveRecord::Import::
|
4
|
+
class ActiveRecord::ConnectionAdapters::Mysql2Adapter
|
5
|
+
include ActiveRecord::Import::Mysql2Adapter
|
6
6
|
end
|
@@ -28,7 +28,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
28
28
|
else
|
29
29
|
select_values( sql2insert, *args )
|
30
30
|
end
|
31
|
-
|
31
|
+
clear_query_cache if query_cache_enabled
|
32
32
|
end
|
33
33
|
|
34
34
|
if options[:returning].blank?
|
@@ -203,8 +203,6 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
203
203
|
true
|
204
204
|
end
|
205
205
|
|
206
|
-
private
|
207
|
-
|
208
206
|
def database_version
|
209
207
|
defined?(postgresql_version) ? postgresql_version : super
|
210
208
|
end
|
@@ -166,8 +166,6 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
166
166
|
exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('duplicate key')
|
167
167
|
end
|
168
168
|
|
169
|
-
private
|
170
|
-
|
171
169
|
def database_version
|
172
170
|
defined?(sqlite_version) ? sqlite_version : super
|
173
171
|
end
|
@@ -34,7 +34,7 @@ module ActiveRecord::Import #:nodoc:
|
|
34
34
|
@validate_callbacks = klass._validate_callbacks.dup
|
35
35
|
|
36
36
|
@validate_callbacks.each_with_index do |callback, i|
|
37
|
-
filter = callback.raw_filter
|
37
|
+
filter = callback.respond_to?(:raw_filter) ? callback.raw_filter : callback.filter
|
38
38
|
next unless filter.class.name =~ /Validations::PresenceValidator/ ||
|
39
39
|
(!@options[:validate_uniqueness] &&
|
40
40
|
filter.is_a?(ActiveRecord::Validations::UniquenessValidator))
|
@@ -49,7 +49,7 @@ module ActiveRecord::Import #:nodoc:
|
|
49
49
|
associations = klass.reflect_on_all_associations(:belongs_to)
|
50
50
|
associations.each do |assoc|
|
51
51
|
if (index = attrs.index(assoc.name))
|
52
|
-
key = assoc.foreign_key.to_sym
|
52
|
+
key = assoc.foreign_key.is_a?(Array) ? assoc.foreign_key.map(&:to_sym) : assoc.foreign_key.to_sym
|
53
53
|
attrs[index] = key unless attrs.include?(key)
|
54
54
|
end
|
55
55
|
end
|
@@ -734,7 +734,10 @@ class ActiveRecord::Base
|
|
734
734
|
set_attributes_and_mark_clean(models, return_obj, timestamps, options)
|
735
735
|
|
736
736
|
# if there are auto-save associations on the models we imported that are new, import them as well
|
737
|
-
|
737
|
+
if options[:recursive]
|
738
|
+
options[:on_duplicate_key_update] = on_duplicate_key_update unless on_duplicate_key_update.nil?
|
739
|
+
import_associations(models, options.dup.merge(validate: false))
|
740
|
+
end
|
738
741
|
end
|
739
742
|
|
740
743
|
return_obj
|
@@ -775,21 +778,22 @@ class ActiveRecord::Base
|
|
775
778
|
unless scope_columns.blank?
|
776
779
|
scope_columns.zip(scope_values).each do |name, value|
|
777
780
|
name_as_sym = name.to_sym
|
778
|
-
next if column_names.include?(name_as_sym)
|
779
|
-
|
780
|
-
is_sti = (name_as_sym == inheritance_column.to_sym && self < base_class)
|
781
|
-
value = Array(value).first if is_sti
|
782
|
-
|
781
|
+
next if column_names.include?(name_as_sym) || name_as_sym == inheritance_column.to_sym
|
783
782
|
column_names << name_as_sym
|
784
783
|
array_of_attributes.each { |attrs| attrs << value }
|
785
784
|
end
|
786
785
|
end
|
787
786
|
|
787
|
+
if finder_needs_type_condition?
|
788
|
+
unless column_names.include?(inheritance_column.to_sym)
|
789
|
+
column_names << inheritance_column.to_sym
|
790
|
+
array_of_attributes.each { |attrs| attrs << sti_name }
|
791
|
+
end
|
792
|
+
end
|
793
|
+
|
788
794
|
columns = column_names.each_with_index.map do |name, i|
|
789
795
|
column = columns_hash[name.to_s]
|
790
|
-
|
791
796
|
raise ActiveRecord::Import::MissingColumnError.new(name.to_s, i) if column.nil?
|
792
|
-
|
793
797
|
column
|
794
798
|
end
|
795
799
|
|
@@ -805,17 +809,29 @@ class ActiveRecord::Base
|
|
805
809
|
if supports_import?
|
806
810
|
# generate the sql
|
807
811
|
post_sql_statements = connection.post_sql_statements( quoted_table_name, options )
|
812
|
+
import_size = values_sql.size
|
813
|
+
|
814
|
+
batch_size = options[:batch_size] || import_size
|
815
|
+
run_proc = options[:batch_size].to_i.positive? && options[:batch_progress].respond_to?( :call )
|
816
|
+
progress_proc = options[:batch_progress]
|
817
|
+
current_batch = 0
|
818
|
+
batches = (import_size / batch_size.to_f).ceil
|
808
819
|
|
809
|
-
batch_size = options[:batch_size] || values_sql.size
|
810
820
|
values_sql.each_slice(batch_size) do |batch_values|
|
821
|
+
batch_started_at = Time.now.to_i
|
822
|
+
|
811
823
|
# perform the inserts
|
812
824
|
result = connection.insert_many( [insert_sql, post_sql_statements].flatten,
|
813
825
|
batch_values,
|
814
826
|
options,
|
815
827
|
"#{model_name} Create Many" )
|
828
|
+
|
816
829
|
number_inserted += result.num_inserts
|
817
830
|
ids += result.ids
|
818
831
|
results += result.results
|
832
|
+
current_batch += 1
|
833
|
+
|
834
|
+
progress_proc.call(import_size, batches, current_batch, Time.now.to_i - batch_started_at) if run_proc
|
819
835
|
end
|
820
836
|
else
|
821
837
|
transaction(requires_new: true) do
|
@@ -841,7 +857,7 @@ class ActiveRecord::Base
|
|
841
857
|
model.id = id
|
842
858
|
|
843
859
|
timestamps.each do |attr, value|
|
844
|
-
model.send(attr + "=", value)
|
860
|
+
model.send(attr + "=", value) if model.send(attr).nil?
|
845
861
|
end
|
846
862
|
end
|
847
863
|
end
|
@@ -895,15 +911,19 @@ class ActiveRecord::Base
|
|
895
911
|
changed_columns = model.changed
|
896
912
|
association_reflections = model.class.reflect_on_all_associations(:belongs_to)
|
897
913
|
association_reflections.each do |association_reflection|
|
898
|
-
column_name = association_reflection.foreign_key
|
899
914
|
next if association_reflection.options[:polymorphic]
|
900
|
-
next if changed_columns.include?(column_name)
|
901
|
-
association = model.association(association_reflection.name)
|
902
|
-
association = association.target
|
903
|
-
next if association.blank? || model.public_send(column_name).present?
|
904
915
|
|
905
|
-
|
906
|
-
|
916
|
+
column_names = Array(association_reflection.foreign_key).map(&:to_s)
|
917
|
+
column_names.each_with_index do |column_name, column_index|
|
918
|
+
next if changed_columns.include?(column_name)
|
919
|
+
|
920
|
+
association = model.association(association_reflection.name)
|
921
|
+
association = association.target
|
922
|
+
next if association.blank? || model.public_send(column_name).present?
|
923
|
+
|
924
|
+
association_primary_key = Array(association_reflection.association_primary_key)[column_index]
|
925
|
+
model.public_send("#{column_name}=", association.send(association_primary_key))
|
926
|
+
end
|
907
927
|
end
|
908
928
|
end
|
909
929
|
|
@@ -916,8 +936,9 @@ class ActiveRecord::Base
|
|
916
936
|
associated_objects_by_class = {}
|
917
937
|
models.each { |model| find_associated_objects_for_import(associated_objects_by_class, model) }
|
918
938
|
|
919
|
-
# :on_duplicate_key_update
|
920
|
-
options.delete(:on_duplicate_key_update)
|
939
|
+
# :on_duplicate_key_update only supported for all fields
|
940
|
+
options.delete(:on_duplicate_key_update) unless options[:on_duplicate_key_update] == :all
|
941
|
+
# :returning not supported for associations
|
921
942
|
options.delete(:returning)
|
922
943
|
|
923
944
|
associated_objects_by_class.each_value do |associations|
|
@@ -949,8 +970,13 @@ class ActiveRecord::Base
|
|
949
970
|
changed_objects.each do |child|
|
950
971
|
child.public_send("#{association_reflection.foreign_key}=", model.id)
|
951
972
|
# For polymorphic associations
|
973
|
+
association_name = if model.class.respond_to?(:polymorphic_name)
|
974
|
+
model.class.polymorphic_name
|
975
|
+
else
|
976
|
+
model.class.base_class
|
977
|
+
end
|
952
978
|
association_reflection.type.try do |type|
|
953
|
-
child.public_send("#{type}=",
|
979
|
+
child.public_send("#{type}=", association_name)
|
954
980
|
end
|
955
981
|
end
|
956
982
|
associated_objects_by_class[model.class.name][association_reflection.name].concat changed_objects
|
@@ -1011,7 +1037,12 @@ class ActiveRecord::Base
|
|
1011
1037
|
end
|
1012
1038
|
|
1013
1039
|
# use tz as set in ActiveRecord::Base
|
1014
|
-
|
1040
|
+
default_timezone = if ActiveRecord.respond_to?(:default_timezone)
|
1041
|
+
ActiveRecord.default_timezone
|
1042
|
+
else
|
1043
|
+
ActiveRecord::Base.default_timezone
|
1044
|
+
end
|
1045
|
+
timestamp = default_timezone == :utc ? Time.now.utc : Time.now
|
1015
1046
|
|
1016
1047
|
[:create, :update].each do |action|
|
1017
1048
|
timestamp_columns[action].each do |column|
|
@@ -39,7 +39,7 @@ module ActiveRecord # :nodoc:
|
|
39
39
|
|
40
40
|
next unless matched_instance
|
41
41
|
|
42
|
-
instance.
|
42
|
+
instance.instance_variable_set :@association_cache, {}
|
43
43
|
instance.send :clear_aggregation_cache if instance.respond_to?(:clear_aggregation_cache, true)
|
44
44
|
instance.instance_variable_set :@attributes, matched_instance.instance_variable_get(:@attributes)
|
45
45
|
|
@@ -1,7 +1,8 @@
|
|
1
1
|
common: &common
|
2
2
|
username: root
|
3
|
-
password:
|
3
|
+
password: root
|
4
4
|
encoding: utf8
|
5
|
+
collation: utf8_general_ci
|
5
6
|
host: localhost
|
6
7
|
database: activerecord_import_test
|
7
8
|
|
@@ -37,6 +38,7 @@ oracle:
|
|
37
38
|
postgresql: &postgresql
|
38
39
|
<<: *common
|
39
40
|
username: postgres
|
41
|
+
password: postgres
|
40
42
|
adapter: postgresql
|
41
43
|
min_messages: warning
|
42
44
|
|
data/test/import_test.rb
CHANGED
@@ -169,7 +169,17 @@ describe "#import" do
|
|
169
169
|
assert_difference "Dictionary.count", +1 do
|
170
170
|
Dictionary.import dictionaries
|
171
171
|
end
|
172
|
-
assert_equal "Dictionary", Dictionary.
|
172
|
+
assert_equal "Dictionary", Dictionary.last.type
|
173
|
+
end
|
174
|
+
|
175
|
+
it "should import arrays successfully" do
|
176
|
+
columns = [:author_name, :title]
|
177
|
+
values = [["Noah Webster", "Webster's Dictionary"]]
|
178
|
+
|
179
|
+
assert_difference "Dictionary.count", +1 do
|
180
|
+
Dictionary.import columns, values
|
181
|
+
end
|
182
|
+
assert_equal "Dictionary", Dictionary.last.type
|
173
183
|
end
|
174
184
|
end
|
175
185
|
|
@@ -405,6 +415,15 @@ describe "#import" do
|
|
405
415
|
assert_equal 3, result.num_inserts if Topic.supports_import?
|
406
416
|
end
|
407
417
|
end
|
418
|
+
|
419
|
+
it "should accept and call an optional callable to run after each batch" do
|
420
|
+
lambda_called = 0
|
421
|
+
|
422
|
+
my_proc = ->(_row_count, _batches, _batch, _duration) { lambda_called += 1 }
|
423
|
+
Topic.import Build(10, :topics), batch_size: 4, batch_progress: my_proc
|
424
|
+
|
425
|
+
assert_equal 3, lambda_called
|
426
|
+
end
|
408
427
|
end
|
409
428
|
|
410
429
|
context "with :synchronize option" do
|
@@ -652,6 +671,14 @@ describe "#import" do
|
|
652
671
|
assert_equal [val1, val2], scope.map(&column).sort
|
653
672
|
end
|
654
673
|
|
674
|
+
context "for cards and decks" do
|
675
|
+
it "works when the polymorphic name is different than base class name" do
|
676
|
+
deck = Deck.create(id: 1, name: 'test')
|
677
|
+
deck.cards.import [:id, :deck_type], [[1, 'PlayingCard']]
|
678
|
+
assert_equal deck.cards.first.deck_type, "PlayingCard"
|
679
|
+
end
|
680
|
+
end
|
681
|
+
|
655
682
|
it "works importing array of hashes" do
|
656
683
|
scope.import [{ column => val1 }, { column => val2 }]
|
657
684
|
|
data/test/models/card.rb
ADDED
data/test/models/deck.rb
ADDED
@@ -52,6 +52,20 @@ ActiveRecord::Schema.define do
|
|
52
52
|
t.string :name
|
53
53
|
end
|
54
54
|
|
55
|
+
create_table :cards, force: :cascade do |t|
|
56
|
+
t.string :name
|
57
|
+
t.string :deck_type
|
58
|
+
t.integer :deck_id
|
59
|
+
end
|
60
|
+
|
61
|
+
create_table :decks, force: :cascade do |t|
|
62
|
+
t.string :name
|
63
|
+
end
|
64
|
+
|
65
|
+
create_table :playing_cards, force: :cascade do |t|
|
66
|
+
t.string :name
|
67
|
+
end
|
68
|
+
|
55
69
|
create_table :books, force: :cascade do |t|
|
56
70
|
t.string :title, null: false
|
57
71
|
t.string :publisher, null: false, default: 'Default Publisher'
|
@@ -191,4 +205,15 @@ ActiveRecord::Schema.define do
|
|
191
205
|
);
|
192
206
|
).split.join(' ').strip
|
193
207
|
end
|
208
|
+
|
209
|
+
create_table :customers, force: :cascade do |t|
|
210
|
+
t.integer :account_id
|
211
|
+
t.string :name
|
212
|
+
end
|
213
|
+
|
214
|
+
create_table :orders, force: :cascade do |t|
|
215
|
+
t.integer :account_id
|
216
|
+
t.integer :customer_id
|
217
|
+
t.integer :amount
|
218
|
+
end
|
194
219
|
end
|
@@ -290,6 +290,30 @@ def should_support_postgresql_import_functionality
|
|
290
290
|
assert_equal(binary_value, Alarm.first.secret_key)
|
291
291
|
end
|
292
292
|
end
|
293
|
+
|
294
|
+
unless ENV["SKIP_COMPOSITE_PK"]
|
295
|
+
describe "with composite foreign keys" do
|
296
|
+
let(:account_id) { 555 }
|
297
|
+
let(:customer) { Customer.new(account_id: account_id, name: "foo") }
|
298
|
+
let(:order) { Order.new(account_id: account_id, amount: 100, customer: customer) }
|
299
|
+
|
300
|
+
it "imports and correctly maps foreign keys" do
|
301
|
+
assert_difference "Customer.count", +1 do
|
302
|
+
Customer.import [customer]
|
303
|
+
end
|
304
|
+
|
305
|
+
assert_difference "Order.count", +1 do
|
306
|
+
Order.import [order]
|
307
|
+
end
|
308
|
+
|
309
|
+
db_customer = Customer.last
|
310
|
+
db_order = Order.last
|
311
|
+
|
312
|
+
assert_equal db_customer.orders.last, db_order
|
313
|
+
assert_not_equal db_order.customer_id, nil
|
314
|
+
end
|
315
|
+
end
|
316
|
+
end
|
293
317
|
end
|
294
318
|
|
295
319
|
def should_support_postgresql_upsert_functionality
|
@@ -176,7 +176,7 @@ def should_support_recursive_import
|
|
176
176
|
end
|
177
177
|
end
|
178
178
|
|
179
|
-
# If adapter supports on_duplicate_key_update, it is only applied to top level models so that SQL with invalid
|
179
|
+
# If adapter supports on_duplicate_key_update and specific columns are specified, it is only applied to top level models so that SQL with invalid
|
180
180
|
# columns, keys, etc isn't generated for child associations when doing recursive import
|
181
181
|
if ActiveRecord::Base.connection.supports_on_duplicate_key_update?
|
182
182
|
describe "on_duplicate_key_update" do
|
@@ -190,6 +190,26 @@ def should_support_recursive_import
|
|
190
190
|
end
|
191
191
|
end
|
192
192
|
end
|
193
|
+
|
194
|
+
context "when :all fields are updated" do
|
195
|
+
setup do
|
196
|
+
Topic.import new_topics, recursive: true
|
197
|
+
end
|
198
|
+
|
199
|
+
it "updates associated objects" do
|
200
|
+
new_author_name = 'Richard Bachman'
|
201
|
+
topic = new_topics.first
|
202
|
+
topic.books.each do |book|
|
203
|
+
book.author_name = new_author_name
|
204
|
+
end
|
205
|
+
assert_nothing_raised do
|
206
|
+
Topic.import new_topics, recursive: true, on_duplicate_key_update: :all
|
207
|
+
end
|
208
|
+
Topic.find(topic.id).books.each do |book|
|
209
|
+
assert_equal new_author_name, book.author_name
|
210
|
+
end
|
211
|
+
end
|
212
|
+
end
|
193
213
|
end
|
194
214
|
end
|
195
215
|
|
data/test/test_helper.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'pathname'
|
2
|
+
require 'rake'
|
2
3
|
test_dir = Pathname.new File.dirname(__FILE__)
|
3
4
|
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
4
5
|
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
@@ -48,7 +49,15 @@ adapter = ENV["ARE_DB"] || "sqlite3"
|
|
48
49
|
FileUtils.mkdir_p 'log'
|
49
50
|
ActiveRecord::Base.logger = Logger.new("log/test.log")
|
50
51
|
ActiveRecord::Base.logger.level = Logger::DEBUG
|
51
|
-
|
52
|
+
|
53
|
+
if ENV['AR_VERSION'].to_f >= 6.0
|
54
|
+
yaml_config = YAML.load_file(test_dir.join("database.yml"))[adapter]
|
55
|
+
config = ActiveRecord::DatabaseConfigurations::HashConfig.new("test", adapter, yaml_config)
|
56
|
+
ActiveRecord::Base.configurations.configurations << config
|
57
|
+
else
|
58
|
+
ActiveRecord::Base.configurations["test"] = YAML.load_file(test_dir.join("database.yml"))[adapter]
|
59
|
+
end
|
60
|
+
|
52
61
|
ActiveRecord::Base.default_timezone = :utc
|
53
62
|
|
54
63
|
require "activerecord-import"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: activerecord-import
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0
|
4
|
+
version: 1.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Zach Dennis
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-12-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
19
|
+
version: '4.2'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
26
|
+
version: '4.2'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: rake
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -45,10 +45,10 @@ executables: []
|
|
45
45
|
extensions: []
|
46
46
|
extra_rdoc_files: []
|
47
47
|
files:
|
48
|
+
- ".github/workflows/test.yaml"
|
48
49
|
- ".gitignore"
|
49
50
|
- ".rubocop.yml"
|
50
51
|
- ".rubocop_todo.yml"
|
51
|
-
- ".travis.yml"
|
52
52
|
- Brewfile
|
53
53
|
- CHANGELOG.md
|
54
54
|
- Gemfile
|
@@ -68,15 +68,13 @@ files:
|
|
68
68
|
- benchmarks/models/test_memory.rb
|
69
69
|
- benchmarks/models/test_myisam.rb
|
70
70
|
- benchmarks/schema/mysql2_schema.rb
|
71
|
-
- gemfiles/3.2.gemfile
|
72
|
-
- gemfiles/4.0.gemfile
|
73
|
-
- gemfiles/4.1.gemfile
|
74
71
|
- gemfiles/4.2.gemfile
|
75
72
|
- gemfiles/5.0.gemfile
|
76
73
|
- gemfiles/5.1.gemfile
|
77
74
|
- gemfiles/5.2.gemfile
|
78
75
|
- gemfiles/6.0.gemfile
|
79
76
|
- gemfiles/6.1.gemfile
|
77
|
+
- gemfiles/7.0.gemfile
|
80
78
|
- lib/activerecord-import.rb
|
81
79
|
- lib/activerecord-import/active_record/adapters/abstract_adapter.rb
|
82
80
|
- lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb
|
@@ -114,6 +112,7 @@ files:
|
|
114
112
|
- test/adapters/spatialite.rb
|
115
113
|
- test/adapters/sqlite3.rb
|
116
114
|
- test/database.yml.sample
|
115
|
+
- test/github/database.yml
|
117
116
|
- test/import_test.rb
|
118
117
|
- test/jdbcmysql/import_test.rb
|
119
118
|
- test/jdbcpostgresql/import_test.rb
|
@@ -125,11 +124,16 @@ files:
|
|
125
124
|
- test/models/bike_maker.rb
|
126
125
|
- test/models/book.rb
|
127
126
|
- test/models/car.rb
|
127
|
+
- test/models/card.rb
|
128
128
|
- test/models/chapter.rb
|
129
|
+
- test/models/customer.rb
|
130
|
+
- test/models/deck.rb
|
129
131
|
- test/models/dictionary.rb
|
130
132
|
- test/models/discount.rb
|
131
133
|
- test/models/end_note.rb
|
132
134
|
- test/models/group.rb
|
135
|
+
- test/models/order.rb
|
136
|
+
- test/models/playing_card.rb
|
133
137
|
- test/models/promotion.rb
|
134
138
|
- test/models/question.rb
|
135
139
|
- test/models/rule.rb
|
@@ -164,14 +168,13 @@ files:
|
|
164
168
|
- test/support/sqlite3/import_examples.rb
|
165
169
|
- test/synchronize_test.rb
|
166
170
|
- test/test_helper.rb
|
167
|
-
- test/travis/database.yml
|
168
171
|
- test/value_sets_bytes_parser_test.rb
|
169
172
|
- test/value_sets_records_parser_test.rb
|
170
173
|
homepage: https://github.com/zdennis/activerecord-import
|
171
174
|
licenses:
|
172
175
|
- MIT
|
173
176
|
metadata: {}
|
174
|
-
post_install_message:
|
177
|
+
post_install_message:
|
175
178
|
rdoc_options: []
|
176
179
|
require_paths:
|
177
180
|
- lib
|
@@ -179,15 +182,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
179
182
|
requirements:
|
180
183
|
- - ">="
|
181
184
|
- !ruby/object:Gem::Version
|
182
|
-
version: 2.
|
185
|
+
version: 2.4.0
|
183
186
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
184
187
|
requirements:
|
185
188
|
- - ">="
|
186
189
|
- !ruby/object:Gem::Version
|
187
190
|
version: '0'
|
188
191
|
requirements: []
|
189
|
-
rubygems_version: 3.0.
|
190
|
-
signing_key:
|
192
|
+
rubygems_version: 3.0.3
|
193
|
+
signing_key:
|
191
194
|
specification_version: 4
|
192
195
|
summary: Bulk insert extension for ActiveRecord
|
193
196
|
test_files:
|
@@ -205,6 +208,7 @@ test_files:
|
|
205
208
|
- test/adapters/spatialite.rb
|
206
209
|
- test/adapters/sqlite3.rb
|
207
210
|
- test/database.yml.sample
|
211
|
+
- test/github/database.yml
|
208
212
|
- test/import_test.rb
|
209
213
|
- test/jdbcmysql/import_test.rb
|
210
214
|
- test/jdbcpostgresql/import_test.rb
|
@@ -216,11 +220,16 @@ test_files:
|
|
216
220
|
- test/models/bike_maker.rb
|
217
221
|
- test/models/book.rb
|
218
222
|
- test/models/car.rb
|
223
|
+
- test/models/card.rb
|
219
224
|
- test/models/chapter.rb
|
225
|
+
- test/models/customer.rb
|
226
|
+
- test/models/deck.rb
|
220
227
|
- test/models/dictionary.rb
|
221
228
|
- test/models/discount.rb
|
222
229
|
- test/models/end_note.rb
|
223
230
|
- test/models/group.rb
|
231
|
+
- test/models/order.rb
|
232
|
+
- test/models/playing_card.rb
|
224
233
|
- test/models/promotion.rb
|
225
234
|
- test/models/question.rb
|
226
235
|
- test/models/rule.rb
|
@@ -255,6 +264,5 @@ test_files:
|
|
255
264
|
- test/support/sqlite3/import_examples.rb
|
256
265
|
- test/synchronize_test.rb
|
257
266
|
- test/test_helper.rb
|
258
|
-
- test/travis/database.yml
|
259
267
|
- test/value_sets_bytes_parser_test.rb
|
260
268
|
- test/value_sets_records_parser_test.rb
|
data/.travis.yml
DELETED
@@ -1,74 +0,0 @@
|
|
1
|
-
language: ruby
|
2
|
-
cache: bundler
|
3
|
-
rvm:
|
4
|
-
- 2.5.5
|
5
|
-
|
6
|
-
env:
|
7
|
-
global:
|
8
|
-
# https://github.com/discourse/discourse/blob/master/.travis.yml
|
9
|
-
- RUBY_GC_MALLOC_LIMIT=50000000
|
10
|
-
matrix:
|
11
|
-
- AR_VERSION=5.1
|
12
|
-
- AR_VERSION=5.2
|
13
|
-
- AR_VERSION=6.0
|
14
|
-
|
15
|
-
matrix:
|
16
|
-
include:
|
17
|
-
- rvm: 2.3.8
|
18
|
-
env: AR_VERSION=3.2
|
19
|
-
- rvm: 2.3.8
|
20
|
-
env: AR_VERSION=4.0
|
21
|
-
- rvm: 2.3.8
|
22
|
-
env: AR_VERSION=4.1
|
23
|
-
- rvm: 2.3.8
|
24
|
-
env: AR_VERSION=4.2
|
25
|
-
- rvm: 2.3.8
|
26
|
-
env: AR_VERSION=5.0
|
27
|
-
|
28
|
-
fast_finish: true
|
29
|
-
|
30
|
-
addons:
|
31
|
-
postgresql: "9.5"
|
32
|
-
apt:
|
33
|
-
sources:
|
34
|
-
- travis-ci/sqlite3
|
35
|
-
- mysql-5.7-trusty
|
36
|
-
packages:
|
37
|
-
- sqlite3
|
38
|
-
- mysql-server
|
39
|
-
- mysql-client
|
40
|
-
- postgresql-9.5-postgis-2.4
|
41
|
-
|
42
|
-
before_install:
|
43
|
-
- gem update --system
|
44
|
-
- sudo mysql -e "use mysql; update user set authentication_string=PASSWORD('') where User='root'; update user set plugin='mysql_native_password';FLUSH PRIVILEGES;"
|
45
|
-
- sudo mysql_upgrade
|
46
|
-
- sudo service mysql restart
|
47
|
-
|
48
|
-
before_script:
|
49
|
-
- mysql -e 'create database activerecord_import_test;'
|
50
|
-
- psql -c 'create database activerecord_import_test;' -U postgres
|
51
|
-
- psql activerecord_import_test -c 'create extension if not exists hstore;' -U postgres
|
52
|
-
- psql -c 'create extension if not exists postgis;' -U postgres
|
53
|
-
- psql -c 'create extension if not exists "uuid-ossp";' -U postgres
|
54
|
-
- cp test/travis/database.yml test/database.yml
|
55
|
-
|
56
|
-
script:
|
57
|
-
- bundle exec rake test:mysql2
|
58
|
-
- bundle exec rake test:mysql2_makara
|
59
|
-
- bundle exec rake test:mysql2spatial
|
60
|
-
- bundle exec rake test:postgis
|
61
|
-
- bundle exec rake test:postgresql
|
62
|
-
- bundle exec rake test:postgresql_makara
|
63
|
-
- bundle exec rake test:seamless_database_pool
|
64
|
-
- bundle exec rake test:spatialite
|
65
|
-
- bundle exec rake test:sqlite3
|
66
|
-
- bundle exec rubocop
|
67
|
-
|
68
|
-
dist: xenial
|
69
|
-
|
70
|
-
services:
|
71
|
-
- mysql
|
72
|
-
- postgresql
|
73
|
-
|
74
|
-
sudo: required
|
data/gemfiles/3.2.gemfile
DELETED
data/gemfiles/4.0.gemfile
DELETED
data/gemfiles/4.1.gemfile
DELETED