activerecord-import 1.4.1 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yaml +73 -15
- data/.gitignore +4 -0
- data/.rubocop.yml +10 -7
- data/.rubocop_todo.yml +10 -16
- data/CHANGELOG.md +77 -1
- data/Dockerfile +23 -0
- data/Gemfile +20 -18
- data/README.markdown +46 -7
- data/Rakefile +5 -0
- data/activerecord-import.gemspec +4 -0
- data/benchmarks/benchmark.rb +3 -5
- data/benchmarks/lib/base.rb +8 -5
- data/benchmarks/lib/cli_parser.rb +9 -7
- data/docker-compose.yml +34 -0
- data/gemfiles/7.1.gemfile +3 -0
- data/gemfiles/7.2.gemfile +3 -0
- data/gemfiles/8.0.gemfile +3 -0
- data/lib/activerecord-import/active_record/adapters/trilogy_adapter.rb +8 -0
- data/lib/activerecord-import/adapters/abstract_adapter.rb +6 -5
- data/lib/activerecord-import/adapters/mysql_adapter.rb +24 -18
- data/lib/activerecord-import/adapters/postgresql_adapter.rb +26 -18
- data/lib/activerecord-import/adapters/sqlite3_adapter.rb +29 -23
- data/lib/activerecord-import/adapters/trilogy_adapter.rb +7 -0
- data/lib/activerecord-import/base.rb +4 -0
- data/lib/activerecord-import/import.rb +64 -36
- data/lib/activerecord-import/value_sets_parser.rb +1 -0
- data/lib/activerecord-import/version.rb +1 -1
- data/lib/activerecord-import.rb +0 -1
- data/test/adapters/janus_mysql2.rb +3 -0
- data/test/adapters/janus_trilogy.rb +11 -0
- data/test/adapters/mysql2_proxy.rb +3 -0
- data/test/adapters/postgresql_proxy.rb +3 -0
- data/test/adapters/trilogy.rb +11 -0
- data/test/database.yml.sample +21 -0
- data/test/github/database.yml +19 -2
- data/test/import_test.rb +15 -39
- data/test/janus_mysql2/import_test.rb +8 -0
- data/test/janus_trilogy/import_test.rb +7 -0
- data/test/jdbcmysql/import_test.rb +3 -3
- data/test/jdbcpostgresql/import_test.rb +2 -2
- data/test/jdbcsqlite3/import_test.rb +2 -2
- data/test/makara_postgis/import_test.rb +2 -2
- data/test/models/author.rb +9 -0
- data/test/models/bike_maker.rb +1 -0
- data/test/models/book.rb +10 -3
- data/test/models/composite_book.rb +19 -0
- data/test/models/composite_chapter.rb +12 -0
- data/test/models/customer.rb +14 -4
- data/test/models/order.rb +13 -4
- data/test/models/tag.rb +6 -1
- data/test/models/tag_alias.rb +7 -1
- data/test/models/topic.rb +6 -0
- data/test/models/widget.rb +10 -3
- data/test/mysql2/import_test.rb +3 -3
- data/test/mysql2_makara/import_test.rb +3 -3
- data/test/mysql2_proxy/import_test.rb +6 -0
- data/test/mysqlspatial2/import_test.rb +3 -3
- data/test/postgis/import_test.rb +2 -2
- data/test/postgresql/import_test.rb +2 -2
- data/test/postgresql_proxy/import_test.rb +6 -0
- data/test/schema/generic_schema.rb +4 -1
- data/test/schema/jdbcpostgresql_schema.rb +1 -1
- data/test/schema/postgis_schema.rb +1 -1
- data/test/schema/postgresql_schema.rb +35 -4
- data/test/sqlite3/import_test.rb +2 -2
- data/test/support/active_support/test_case_extensions.rb +1 -5
- data/test/support/mysql/import_examples.rb +6 -8
- data/test/support/postgresql/import_examples.rb +49 -53
- data/test/support/shared_examples/on_duplicate_key_update.rb +67 -10
- data/test/support/shared_examples/recursive_import.rb +106 -1
- data/test/test_helper.rb +13 -22
- data/test/trilogy/import_test.rb +7 -0
- data/test/value_sets_bytes_parser_test.rb +1 -1
- data/test/value_sets_records_parser_test.rb +1 -1
- metadata +37 -9
- data/gemfiles/4.2.gemfile +0 -4
- data/gemfiles/5.0.gemfile +0 -4
- data/gemfiles/5.1.gemfile +0 -4
- data/lib/activerecord-import/mysql2.rb +0 -9
- data/lib/activerecord-import/postgresql.rb +0 -9
- data/lib/activerecord-import/sqlite3.rb +0 -9
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f7cd19fa557186daf34db96db71f26a7db70b15e66a0a42eaab33be304945ab0
|
|
4
|
+
data.tar.gz: ff8afe763d437210ea1c78e21bacd0b4ae84f113fbadbda087077bf402f08e64
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 402d2087c8c35a2adfa3787ee4e183883df74cc1f4ad2276de1227ba9778b630bfbee546faa3b53b7dcbe1142232bd7d03de046ccab5c6caa5c475a3bc4f15c1
|
|
7
|
+
data.tar.gz: ef5ce7d4dba744b3c58a0affbe326b50d764c7e4c4726b468ad4d18eacee13fbaa8d7b5d78459bc048e8eeb0628cf34b52a59f759f0a1f4d9d491b7439d5845a
|
data/.github/workflows/test.yaml
CHANGED
|
@@ -16,23 +16,71 @@ jobs:
|
|
|
16
16
|
--health-interval 10s
|
|
17
17
|
--health-timeout 5s
|
|
18
18
|
--health-retries 5
|
|
19
|
+
mysql:
|
|
20
|
+
image: mysql:5.7
|
|
21
|
+
ports:
|
|
22
|
+
- 3306:3306
|
|
23
|
+
env:
|
|
24
|
+
MYSQL_HOST: 127.0.0.1
|
|
25
|
+
MYSQL_ROOT_PASSWORD: root
|
|
26
|
+
MYSQL_USER: github
|
|
27
|
+
MYSQL_PASSWORD: github
|
|
28
|
+
MYSQL_DATABASE: activerecord_import_test
|
|
29
|
+
options: >-
|
|
30
|
+
--health-cmd "mysqladmin ping -h localhost"
|
|
31
|
+
--health-interval 10s
|
|
32
|
+
--health-timeout 5s
|
|
33
|
+
--health-retries 5
|
|
19
34
|
strategy:
|
|
20
35
|
fail-fast: false
|
|
21
36
|
matrix:
|
|
22
37
|
ruby:
|
|
23
|
-
- 3.
|
|
38
|
+
- 3.3
|
|
24
39
|
env:
|
|
40
|
+
- AR_VERSION: '8.0'
|
|
41
|
+
RUBYOPT: --enable-frozen-string-literal
|
|
42
|
+
- AR_VERSION: '7.2'
|
|
43
|
+
RUBYOPT: --enable-frozen-string-literal
|
|
44
|
+
- AR_VERSION: '7.1'
|
|
45
|
+
RUBYOPT: --enable-frozen-string-literal
|
|
25
46
|
- AR_VERSION: '7.0'
|
|
26
47
|
RUBYOPT: --enable-frozen-string-literal
|
|
27
48
|
- AR_VERSION: 6.1
|
|
28
49
|
RUBYOPT: --enable-frozen-string-literal
|
|
29
50
|
include:
|
|
51
|
+
- ruby: 3.2
|
|
52
|
+
env:
|
|
53
|
+
AR_VERSION: '8.0'
|
|
54
|
+
- ruby: 3.2
|
|
55
|
+
env:
|
|
56
|
+
AR_VERSION: '7.2'
|
|
57
|
+
- ruby: 3.2
|
|
58
|
+
env:
|
|
59
|
+
AR_VERSION: '7.1'
|
|
60
|
+
- ruby: 3.2
|
|
61
|
+
env:
|
|
62
|
+
AR_VERSION: '7.0'
|
|
63
|
+
- ruby: 3.2
|
|
64
|
+
env:
|
|
65
|
+
AR_VERSION: 6.1
|
|
66
|
+
- ruby: 3.1
|
|
67
|
+
env:
|
|
68
|
+
AR_VERSION: '7.1'
|
|
69
|
+
- ruby: 3.1
|
|
70
|
+
env:
|
|
71
|
+
AR_VERSION: '7.0'
|
|
72
|
+
- ruby: 3.1
|
|
73
|
+
env:
|
|
74
|
+
AR_VERSION: 6.1
|
|
30
75
|
- ruby: '3.0'
|
|
31
76
|
env:
|
|
32
77
|
AR_VERSION: '7.0'
|
|
33
78
|
- ruby: '3.0'
|
|
34
79
|
env:
|
|
35
80
|
AR_VERSION: 6.1
|
|
81
|
+
- ruby: jruby-9.4.8.0
|
|
82
|
+
env:
|
|
83
|
+
AR_VERSION: '7.0'
|
|
36
84
|
- ruby: 2.7
|
|
37
85
|
env:
|
|
38
86
|
AR_VERSION: '7.0'
|
|
@@ -42,32 +90,29 @@ jobs:
|
|
|
42
90
|
- ruby: 2.7
|
|
43
91
|
env:
|
|
44
92
|
AR_VERSION: '6.0'
|
|
45
|
-
- ruby:
|
|
93
|
+
- ruby: jruby-9.3.15.0
|
|
46
94
|
env:
|
|
47
|
-
AR_VERSION:
|
|
95
|
+
AR_VERSION: '6.1'
|
|
48
96
|
- ruby: 2.6
|
|
49
97
|
env:
|
|
50
|
-
AR_VERSION: 5.
|
|
51
|
-
- ruby: 2.4
|
|
52
|
-
env:
|
|
53
|
-
AR_VERSION: '5.0'
|
|
54
|
-
- ruby: 2.4
|
|
55
|
-
env:
|
|
56
|
-
AR_VERSION: 4.2
|
|
98
|
+
AR_VERSION: 5.2
|
|
57
99
|
runs-on: ubuntu-latest
|
|
58
100
|
env:
|
|
59
101
|
AR_VERSION: ${{ matrix.env.AR_VERSION }}
|
|
60
102
|
DB_DATABASE: activerecord_import_test
|
|
61
103
|
steps:
|
|
62
|
-
- uses: actions/checkout@
|
|
104
|
+
- uses: actions/checkout@v4
|
|
105
|
+
- name: Install SQLite3 Development Library
|
|
106
|
+
run: |
|
|
107
|
+
sudo apt-get update
|
|
108
|
+
sudo apt-get install libsqlite3-dev
|
|
63
109
|
- uses: ruby/setup-ruby@v1
|
|
64
110
|
with:
|
|
65
111
|
ruby-version: ${{ matrix.ruby }}
|
|
66
112
|
bundler-cache: true
|
|
113
|
+
rubygems: latest
|
|
67
114
|
- name: Set up databases
|
|
68
115
|
run: |
|
|
69
|
-
sudo /etc/init.d/mysql start
|
|
70
|
-
mysql -e 'CREATE DATABASE ${{ env.DB_DATABASE }} CHARACTER SET utf8 COLLATE utf8_general_ci;' -u root -proot
|
|
71
116
|
psql -h localhost -U postgres -c 'create database ${{ env.DB_DATABASE }};'
|
|
72
117
|
psql -h localhost -U postgres -d ${{ env.DB_DATABASE }} -c 'create extension if not exists hstore;'
|
|
73
118
|
psql -h localhost -U postgres -c 'create extension if not exists postgis;'
|
|
@@ -80,11 +125,14 @@ jobs:
|
|
|
80
125
|
bundle exec rake test:mysql2
|
|
81
126
|
bundle exec rake test:mysql2_makara
|
|
82
127
|
bundle exec rake test:mysql2spatial
|
|
128
|
+
bundle exec rake test:mysql2_proxy
|
|
129
|
+
bundle exec rake test:janus_mysql2
|
|
83
130
|
- name: Run tests with postgresql
|
|
84
131
|
run: |
|
|
85
132
|
bundle exec rake test:postgis
|
|
86
133
|
bundle exec rake test:postgresql
|
|
87
134
|
bundle exec rake test:postgresql_makara
|
|
135
|
+
bundle exec rake test:postgresql_proxy
|
|
88
136
|
- name: Run tests with seamless_database_pool
|
|
89
137
|
run: |
|
|
90
138
|
bundle exec rake test:seamless_database_pool
|
|
@@ -93,15 +141,25 @@ jobs:
|
|
|
93
141
|
run: |
|
|
94
142
|
bundle exec rake test:spatialite
|
|
95
143
|
bundle exec rake test:sqlite3
|
|
144
|
+
- name: Run trilogy tests
|
|
145
|
+
if: ${{ matrix.env.AR_VERSION >= '7.0' && !startsWith(matrix.ruby, 'jruby') }}
|
|
146
|
+
run: |
|
|
147
|
+
bundle exec rake test:trilogy
|
|
148
|
+
bundle exec rake test:janus_trilogy
|
|
149
|
+
|
|
96
150
|
lint:
|
|
97
151
|
runs-on: ubuntu-latest
|
|
98
152
|
env:
|
|
99
153
|
AR_VERSION: '7.0'
|
|
100
154
|
steps:
|
|
101
|
-
- uses: actions/checkout@
|
|
155
|
+
- uses: actions/checkout@v4
|
|
156
|
+
- name: Install SQLite3 Development Library
|
|
157
|
+
run: |
|
|
158
|
+
sudo apt-get update
|
|
159
|
+
sudo apt-get install libsqlite3-dev
|
|
102
160
|
- uses: ruby/setup-ruby@v1
|
|
103
161
|
with:
|
|
104
|
-
ruby-version:
|
|
162
|
+
ruby-version: 3.0
|
|
105
163
|
bundler-cache: true
|
|
106
164
|
- name: Run Rubocop
|
|
107
165
|
run: bundle exec rubocop
|
data/.gitignore
CHANGED
data/.rubocop.yml
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
inherit_from: .rubocop_todo.yml
|
|
2
2
|
|
|
3
|
+
AllCops:
|
|
4
|
+
TargetRubyVersion: 3.0.x
|
|
5
|
+
|
|
3
6
|
Metrics/AbcSize:
|
|
4
7
|
Enabled: false
|
|
5
8
|
|
|
@@ -12,9 +15,6 @@ Metrics/ClassLength:
|
|
|
12
15
|
Metrics/CyclomaticComplexity:
|
|
13
16
|
Enabled: false
|
|
14
17
|
|
|
15
|
-
Metrics/LineLength:
|
|
16
|
-
Enabled: false
|
|
17
|
-
|
|
18
18
|
Metrics/MethodLength:
|
|
19
19
|
Enabled: false
|
|
20
20
|
|
|
@@ -66,10 +66,10 @@ Style/SymbolArray:
|
|
|
66
66
|
Style/TrailingCommaInArrayLiteral:
|
|
67
67
|
Enabled: false
|
|
68
68
|
|
|
69
|
-
Layout/
|
|
69
|
+
Layout/ArgumentAlignment:
|
|
70
70
|
Enabled: false
|
|
71
71
|
|
|
72
|
-
Layout/
|
|
72
|
+
Layout/ParameterAlignment:
|
|
73
73
|
EnforcedStyle: with_fixed_indentation
|
|
74
74
|
|
|
75
75
|
Layout/EndAlignment:
|
|
@@ -81,7 +81,7 @@ Layout/ElseAlignment:
|
|
|
81
81
|
Layout/EmptyLineAfterGuardClause:
|
|
82
82
|
Enabled: false
|
|
83
83
|
|
|
84
|
-
Layout/
|
|
84
|
+
Layout/HeredocIndentation:
|
|
85
85
|
Enabled: false
|
|
86
86
|
|
|
87
87
|
Layout/SpaceInsideParens:
|
|
@@ -90,6 +90,9 @@ Layout/SpaceInsideParens:
|
|
|
90
90
|
Layout/SpaceInsidePercentLiteralDelimiters:
|
|
91
91
|
Enabled: false
|
|
92
92
|
|
|
93
|
+
Layout/LineLength:
|
|
94
|
+
Enabled: false
|
|
95
|
+
|
|
93
96
|
Lint/ErbNewArguments:
|
|
94
97
|
Enabled: false
|
|
95
98
|
|
|
@@ -102,7 +105,7 @@ Lint/PercentStringArray:
|
|
|
102
105
|
Naming/HeredocDelimiterNaming:
|
|
103
106
|
Enabled: false
|
|
104
107
|
|
|
105
|
-
Naming/
|
|
108
|
+
Naming/MethodParameterName:
|
|
106
109
|
Enabled: false
|
|
107
110
|
|
|
108
111
|
Security/YAMLLoad:
|
data/.rubocop_todo.yml
CHANGED
|
@@ -1,36 +1,30 @@
|
|
|
1
1
|
# This configuration was generated by
|
|
2
2
|
# `rubocop --auto-gen-config`
|
|
3
|
-
# on
|
|
3
|
+
# on 2023-02-15 00:58:14 UTC using RuboCop version 1.45.1.
|
|
4
4
|
# The point is for the user to remove these configuration records
|
|
5
5
|
# one by one as the offenses are removed from the code base.
|
|
6
6
|
# Note that changes in the inspected code, or installation of new
|
|
7
7
|
# versions of RuboCop, may require this file to be generated again.
|
|
8
8
|
|
|
9
|
-
# Offense count: 2
|
|
10
|
-
Lint/HandleExceptions:
|
|
11
|
-
Exclude:
|
|
12
|
-
- 'lib/activerecord-import/base.rb'
|
|
13
|
-
- 'test/import_test.rb'
|
|
14
|
-
|
|
15
9
|
# Offense count: 2
|
|
16
10
|
Lint/RescueException:
|
|
17
11
|
Exclude:
|
|
18
12
|
- 'benchmarks/lib/cli_parser.rb'
|
|
19
13
|
- 'test/import_test.rb'
|
|
20
14
|
|
|
21
|
-
# Offense count:
|
|
22
|
-
#
|
|
23
|
-
# Configuration parameters: AllowUnusedKeywordArguments, IgnoreEmptyMethods.
|
|
15
|
+
# Offense count: 3
|
|
16
|
+
# This cop supports safe autocorrection (--autocorrect).
|
|
17
|
+
# Configuration parameters: AllowUnusedKeywordArguments, IgnoreEmptyMethods, IgnoreNotImplementedMethods.
|
|
24
18
|
Lint/UnusedMethodArgument:
|
|
25
19
|
Exclude:
|
|
26
20
|
- 'lib/activerecord-import/adapters/postgresql_adapter.rb'
|
|
27
21
|
- 'lib/activerecord-import/import.rb'
|
|
28
22
|
|
|
29
23
|
# Offense count: 2
|
|
30
|
-
|
|
31
|
-
# Configuration parameters: Keywords.
|
|
32
|
-
# Keywords: TODO, FIXME, OPTIMIZE, HACK, REVIEW
|
|
33
|
-
Style/CommentAnnotation:
|
|
24
|
+
Style/CombinableLoops:
|
|
34
25
|
Exclude:
|
|
35
|
-
- '
|
|
36
|
-
|
|
26
|
+
- 'test/support/shared_examples/recursive_import.rb'
|
|
27
|
+
|
|
28
|
+
Naming/FileName:
|
|
29
|
+
Exclude:
|
|
30
|
+
- 'lib/activerecord-import.rb'
|
data/CHANGELOG.md
CHANGED
|
@@ -1,8 +1,84 @@
|
|
|
1
|
+
## Changes in 2.2.0
|
|
2
|
+
|
|
3
|
+
### New Features
|
|
4
|
+
|
|
5
|
+
* Add Support for Janus ActiveRecord adapters. Thanks to @permidon via #\873.
|
|
6
|
+
|
|
7
|
+
## Changes in 2.1.0
|
|
8
|
+
|
|
9
|
+
### New Features
|
|
10
|
+
|
|
11
|
+
* Add Support for `active_record_proxy_adapters` gem thanks to @stingrayzboy via #\867.
|
|
12
|
+
Since Rails 7.1 makara no longer works and it is not currently maintained. The @nasdaq team
|
|
13
|
+
have written a gem called [active_record_proxy_adapters](https://rubygems.org/gems/active_record_proxy_adapters)
|
|
14
|
+
that implements some makara functionality.
|
|
15
|
+
|
|
16
|
+
## Changes in 2.0.0
|
|
17
|
+
|
|
18
|
+
### Breaking Changes
|
|
19
|
+
|
|
20
|
+
* Fix `recursive_on_duplicate_key_update` doesn't work with non-standard
|
|
21
|
+
association name. Thanks to @jacob-carlborg-apoex via \#852. The documentation for the
|
|
22
|
+
`:recursive_on_duplicate_key_update` option specifies that the hash key is
|
|
23
|
+
the association name. But previously the name of associated table was used to
|
|
24
|
+
look up the options. Now the behavior matches the documentation and the name
|
|
25
|
+
of the association is used instead. This only affects associations that uses
|
|
26
|
+
a name that doesn't follow the ActiveRecord naming conventions of
|
|
27
|
+
associations and class names, i.e. when the `class_name:` option is used on
|
|
28
|
+
an association.
|
|
29
|
+
|
|
30
|
+
## Changes in 1.8.1
|
|
31
|
+
|
|
32
|
+
### Fixes
|
|
33
|
+
|
|
34
|
+
* Further update for ActiveRecord 7.2 compatibility when running validations. Thanks to @denisahearn via \##847.
|
|
35
|
+
|
|
36
|
+
## Changes in 1.8.0
|
|
37
|
+
|
|
38
|
+
### New Features
|
|
39
|
+
|
|
40
|
+
* Add support for ActiveRecord 7.2 via \##845.
|
|
41
|
+
|
|
42
|
+
## Changes in 1.7.0
|
|
43
|
+
|
|
44
|
+
### New Features
|
|
45
|
+
|
|
46
|
+
* Add support for ActiveRecord 7.1 composite primary keys. Thanks to @fragkakis via \##837.
|
|
47
|
+
* Add support for upserting associations when doing recursive imports. Thanks to @ramblex via \##778.
|
|
48
|
+
|
|
49
|
+
## Changes in 1.6.0
|
|
50
|
+
|
|
51
|
+
### New Features
|
|
52
|
+
|
|
53
|
+
* Add trilogy adapter support. Thanks to @zmariscal via \##825.
|
|
54
|
+
|
|
55
|
+
### Fixes
|
|
56
|
+
|
|
57
|
+
* Use the locking_enabled? method provided by activerecord to decide whether the lock field should be updated. Thanks to @dombesz via \##822.
|
|
58
|
+
|
|
59
|
+
## Changes in 1.5.1
|
|
60
|
+
|
|
61
|
+
### Fixes
|
|
62
|
+
|
|
63
|
+
* Stop memoizing schema_columns_hash so dynamic schema changes are picked up. Thanks to @koshigoe via \##812.
|
|
64
|
+
|
|
65
|
+
## Changes in 1.5.0
|
|
66
|
+
|
|
67
|
+
### New Features
|
|
68
|
+
|
|
69
|
+
* Add Rails 7.1 support. Thanks to @gucki via \##807.
|
|
70
|
+
* Add support for alias attributes. Thanks to @leonidkroka via \##799.
|
|
71
|
+
|
|
72
|
+
### Fixes
|
|
73
|
+
|
|
74
|
+
* Support for multi-byte column names when splitting queries. Thanks to @TakuyaKurimoto via \##801.
|
|
75
|
+
* Fix issue with track_validation_failures when import models. Thanks to @OtaYohihiro via \##798.
|
|
76
|
+
|
|
1
77
|
## Changes in 1.4.1
|
|
2
78
|
|
|
3
79
|
### Fixes
|
|
4
80
|
|
|
5
|
-
* Fix importing models that have required belongs_to associations and use composite primary keys. Thanks to @thoughtbot-summer
|
|
81
|
+
* Fix importing models that have required belongs_to associations and use composite primary keys. Thanks to @thoughtbot-summer via \##783.
|
|
6
82
|
|
|
7
83
|
## Changes in 1.4.0
|
|
8
84
|
|
data/Dockerfile
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Use the official Ruby 3.2 image as a base image
|
|
2
|
+
ARG RUBY_VERSION=3.2
|
|
3
|
+
FROM ruby:${RUBY_VERSION}-bullseye
|
|
4
|
+
|
|
5
|
+
# Set the working directory
|
|
6
|
+
WORKDIR /usr/src/app
|
|
7
|
+
|
|
8
|
+
# Install system packages
|
|
9
|
+
RUN apt-get update -qq && \
|
|
10
|
+
apt-get install -y default-mysql-client postgresql postgresql-contrib vim && \
|
|
11
|
+
apt-get clean
|
|
12
|
+
|
|
13
|
+
# Set environment variables
|
|
14
|
+
ENV AR_VERSION=7.0
|
|
15
|
+
|
|
16
|
+
# Copy all files
|
|
17
|
+
COPY . .
|
|
18
|
+
|
|
19
|
+
# Move sample database.yml and install gems
|
|
20
|
+
RUN mv test/database.yml.sample test/database.yml && \
|
|
21
|
+
bundle install
|
|
22
|
+
|
|
23
|
+
CMD ["irb"]
|
data/Gemfile
CHANGED
|
@@ -6,16 +6,18 @@ gemspec
|
|
|
6
6
|
|
|
7
7
|
version = ENV['AR_VERSION'].to_f
|
|
8
8
|
|
|
9
|
-
mysql2_version = '0.
|
|
10
|
-
mysql2_version = '0.4.0' if version >= 4.2
|
|
9
|
+
mysql2_version = '0.4.0'
|
|
11
10
|
mysql2_version = '0.5.0' if version >= 6.1
|
|
11
|
+
mysql2_version = '0.5.6' if version >= 8.0
|
|
12
12
|
sqlite3_version = '1.3.0'
|
|
13
13
|
sqlite3_version = '1.4.0' if version >= 6.0
|
|
14
|
+
sqlite3_version = '2.2.0' if version >= 8.0
|
|
14
15
|
pg_version = '0.9'
|
|
15
16
|
pg_version = '1.1' if version >= 6.1
|
|
17
|
+
pg_version = '1.5' if version >= 8.0
|
|
16
18
|
|
|
17
19
|
group :development, :test do
|
|
18
|
-
gem 'rubocop'
|
|
20
|
+
gem 'rubocop'
|
|
19
21
|
gem 'rake'
|
|
20
22
|
end
|
|
21
23
|
|
|
@@ -26,36 +28,36 @@ platforms :ruby do
|
|
|
26
28
|
gem "sqlite3", "~> #{sqlite3_version}"
|
|
27
29
|
# seamless_database_pool requires Ruby ~> 2.0
|
|
28
30
|
gem "seamless_database_pool", "~> 1.0.20" if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('3.0.0')
|
|
31
|
+
gem "trilogy" if version >= 6.0
|
|
32
|
+
if version >= 6.0 && version <= 7.0
|
|
33
|
+
gem "activerecord-trilogy-adapter"
|
|
34
|
+
end
|
|
29
35
|
end
|
|
30
36
|
|
|
31
37
|
platforms :jruby do
|
|
32
38
|
gem "jdbc-mysql"
|
|
33
39
|
gem "jdbc-postgres"
|
|
34
|
-
gem "activerecord-jdbcsqlite3-adapter"
|
|
35
|
-
gem "activerecord-jdbcmysql-adapter"
|
|
36
|
-
gem "activerecord-jdbcpostgresql-adapter"
|
|
40
|
+
gem "activerecord-jdbcsqlite3-adapter"
|
|
41
|
+
gem "activerecord-jdbcmysql-adapter"
|
|
42
|
+
gem "activerecord-jdbcpostgresql-adapter"
|
|
37
43
|
end
|
|
38
44
|
|
|
39
45
|
# Support libs
|
|
40
|
-
|
|
46
|
+
if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.0.0")
|
|
47
|
+
gem "factory_bot"
|
|
48
|
+
else
|
|
49
|
+
gem "factory_bot", "~> 5", "< 6.4.5"
|
|
50
|
+
end
|
|
41
51
|
gem "timecop"
|
|
42
52
|
gem "chronic"
|
|
43
|
-
gem "mocha", "~> 1.
|
|
53
|
+
gem "mocha", "~> 2.1.0"
|
|
44
54
|
|
|
45
55
|
# Debugging
|
|
46
|
-
platforms :jruby do
|
|
47
|
-
gem "ruby-debug", "= 0.10.4"
|
|
48
|
-
end
|
|
49
|
-
|
|
50
56
|
platforms :ruby do
|
|
51
57
|
gem "pry-byebug"
|
|
52
|
-
gem "pry", "~> 0.
|
|
58
|
+
gem "pry", "~> 0.14.0"
|
|
53
59
|
end
|
|
54
60
|
|
|
55
|
-
|
|
56
|
-
gem "minitest"
|
|
57
|
-
else
|
|
58
|
-
gem "test-unit"
|
|
59
|
-
end
|
|
61
|
+
gem "minitest"
|
|
60
62
|
|
|
61
63
|
eval_gemfile File.expand_path("../gemfiles/#{version}.gemfile", __FILE__)
|
data/README.markdown
CHANGED
|
@@ -245,8 +245,8 @@ Book.import columns, books, batch_size: 2, batch_progress: my_proc
|
|
|
245
245
|
|
|
246
246
|
#### Recursive
|
|
247
247
|
|
|
248
|
-
|
|
249
|
-
hashes or arrays as recursive inputs.
|
|
248
|
+
> **Note**
|
|
249
|
+
> This only works with PostgreSQL and ActiveRecord objects. This won't work with hashes or arrays as recursive inputs.
|
|
250
250
|
|
|
251
251
|
Assume that Books <code>has_many</code> Reviews.
|
|
252
252
|
|
|
@@ -265,7 +265,7 @@ Book.import books, recursive: true
|
|
|
265
265
|
Key | Options | Default | Description
|
|
266
266
|
------------------------- | --------------------- | ------------------ | -----------
|
|
267
267
|
:validate | `true`/`false` | `true` | Whether or not to run `ActiveRecord` validations (uniqueness skipped). This option will always be true when using `import!`.
|
|
268
|
-
:validate_uniqueness | `true`/`false` | `false` | Whether or not to run uniqueness validations
|
|
268
|
+
:validate_uniqueness | `true`/`false` | `false` | Whether or not to run ActiveRecord uniqueness validations. Beware this will incur an sql query per-record (N+1 queries). (requires `>= v0.27.0`).
|
|
269
269
|
:validate_with_context | `Symbol` |`:create`/`:update` | Allows passing an ActiveModel validation context for each model. Default is `:create` for new records and `:update` for existing ones.
|
|
270
270
|
:track_validation_failures| `true`/`false` | `false` | When this is set to true, `failed_instances` will be an array of arrays, with each inner array having the form `[:index_in_dataset, :object_with_errors]`
|
|
271
271
|
:on_duplicate_key_ignore | `true`/`false` | `false` | Allows skipping records with duplicate keys. See [here](#duplicate-key-ignore) for more details.
|
|
@@ -274,6 +274,7 @@ Key | Options | Default | Descrip
|
|
|
274
274
|
:synchronize | `Array` | N/A | An array of ActiveRecord instances. This synchronizes existing instances in memory with updates from the import.
|
|
275
275
|
:timestamps | `true`/`false` | `true` | Enables/disables timestamps on imported records.
|
|
276
276
|
:recursive | `true`/`false` | `false` | Imports has_many/has_one associations (PostgreSQL only).
|
|
277
|
+
:recursive_on_duplicate_key_update | `Hash` | N/A | Allows upsert logic to be used for recursive associations. The hash key is the association name and the value has the same options as `:on_duplicate_key_update`. See [here](#duplicate-key-update) for more details.
|
|
277
278
|
:batch_size | `Integer` | total # of records | Max number of records to insert per import
|
|
278
279
|
:raise_error | `true`/`false` | `false` | Raises an exception at the first invalid record. This means there will not be a result object returned. The `import!` method is a shortcut for this.
|
|
279
280
|
:all_or_none | `true`/`false` | `false` | Will not import any records if there is a record with validation errors.
|
|
@@ -364,6 +365,29 @@ book.reload.title # => "Book1" (stayed the same)
|
|
|
364
365
|
book.reload.author # => "Bob Barker" (changed)
|
|
365
366
|
```
|
|
366
367
|
|
|
368
|
+
PostgreSQL Using partial indexes
|
|
369
|
+
|
|
370
|
+
```ruby
|
|
371
|
+
book = Book.create! title: "Book1", author: "George Orwell", published_at: Time.now
|
|
372
|
+
book.author = "Bob Barker"
|
|
373
|
+
|
|
374
|
+
# in migration
|
|
375
|
+
execute <<-SQL
|
|
376
|
+
CREATE INDEX books_published_at_index ON books (published_at) WHERE published_at IS NOT NULL;
|
|
377
|
+
SQL
|
|
378
|
+
|
|
379
|
+
# PostgreSQL version
|
|
380
|
+
Book.import [book], on_duplicate_key_update: {
|
|
381
|
+
conflict_target: [:id],
|
|
382
|
+
index_predicate: "published_at IS NOT NULL",
|
|
383
|
+
columns: [:author]
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
book.reload.title # => "Book1" (stayed the same)
|
|
387
|
+
book.reload.author # => "Bob Barker" (changed)
|
|
388
|
+
book.reload.published_at # => 2017-10-09 (stayed the same)
|
|
389
|
+
```
|
|
390
|
+
|
|
367
391
|
PostgreSQL Using constraints
|
|
368
392
|
|
|
369
393
|
```ruby
|
|
@@ -434,7 +458,8 @@ Should you wish to specify those columns, you may use the option `timestamps: fa
|
|
|
434
458
|
However, it is also possible to set just `:created_at` in specific records. In this case despite using `timestamps: true`, `:created_at` will be updated only in records where that field is `nil`. Same rule applies for record associations when enabling the option `recursive: true`.
|
|
435
459
|
|
|
436
460
|
If you are using custom time zones, these will be respected when performing imports as well as long as `ActiveRecord::Base.default_timezone` is set, which for practically all Rails apps it is.
|
|
437
|
-
|
|
461
|
+
> **Note**
|
|
462
|
+
> If you are using ActiveRecord 7.0 or later, please use `ActiveRecord.default_timezone` instead.
|
|
438
463
|
|
|
439
464
|
### Callbacks
|
|
440
465
|
|
|
@@ -506,7 +531,8 @@ This allows an external gem to dynamically add an adapter without the need to ad
|
|
|
506
531
|
|
|
507
532
|
### Requiring
|
|
508
533
|
|
|
509
|
-
Note
|
|
534
|
+
> **Note**
|
|
535
|
+
> These instructions will only work if you are using version 0.2.0 or higher.
|
|
510
536
|
|
|
511
537
|
#### Autoloading via Bundler
|
|
512
538
|
|
|
@@ -543,11 +569,11 @@ require 'activerecord-import'
|
|
|
543
569
|
### Load Path Setup
|
|
544
570
|
To understand how rubygems loads code you can reference the following:
|
|
545
571
|
|
|
546
|
-
|
|
572
|
+
https://guides.rubygems.org/patterns/#loading-code
|
|
547
573
|
|
|
548
574
|
And an example of how active_record dynamically load adapters:
|
|
549
575
|
|
|
550
|
-
https://github.com/rails/rails/blob/
|
|
576
|
+
https://github.com/rails/rails/blob/main/activerecord/lib/active_record/connection_adapters.rb
|
|
551
577
|
|
|
552
578
|
In summary, when a gem is loaded rubygems adds the `lib` folder of the gem to the global load path `$LOAD_PATH` so that all `require` lookups will not propagate through all of the folders on the load path. When a `require` is issued each folder on the `$LOAD_PATH` is checked for the file and/or folder referenced. This allows a gem (like activerecord-import) to define push the activerecord-import folder (or namespace) on the `$LOAD_PATH` and any adapters provided by activerecord-import will be found by rubygems when the require is issued.
|
|
553
579
|
|
|
@@ -624,6 +650,19 @@ AR_VERSION=7.0 bundle exec rake test:postgresql test:sqlite3 test:mysql2
|
|
|
624
650
|
|
|
625
651
|
Once you have pushed up your changes, you can find your CI results [here](https://github.com/zdennis/activerecord-import/actions).
|
|
626
652
|
|
|
653
|
+
#### Docker Setup
|
|
654
|
+
|
|
655
|
+
Before you begin, make sure you have [Docker](https://www.docker.com/products/docker-desktop/) and [Docker Compose](https://docs.docker.com/compose/) installed on your machine. If you don't, you can install both via Homebrew using the following command:
|
|
656
|
+
|
|
657
|
+
```bash
|
|
658
|
+
brew install docker && brew install docker-compose
|
|
659
|
+
```
|
|
660
|
+
##### Steps
|
|
661
|
+
|
|
662
|
+
1. In your terminal run `docker-compose up --build`
|
|
663
|
+
1. In another tab/window run `docker-compose exec app bash`
|
|
664
|
+
1. In that same terminal run the mysql2 test by running `bundle exec rake test:mysql2`
|
|
665
|
+
|
|
627
666
|
## Issue Triage [](https://www.codetriage.com/zdennis/activerecord-import)
|
|
628
667
|
|
|
629
668
|
You can triage issues which may include reproducing bug reports or asking for vital information, such as version numbers or reproduction instructions. If you would like to start triaging issues, one easy way to get started is to [subscribe to activerecord-import on CodeTriage](https://www.codetriage.com/zdennis/activerecord-import).
|
data/Rakefile
CHANGED
|
@@ -19,16 +19,21 @@ ADAPTERS = %w(
|
|
|
19
19
|
mysql2
|
|
20
20
|
mysql2_makara
|
|
21
21
|
mysql2spatial
|
|
22
|
+
mysql2_proxy
|
|
23
|
+
janus_mysql2
|
|
22
24
|
jdbcmysql
|
|
23
25
|
jdbcsqlite3
|
|
24
26
|
jdbcpostgresql
|
|
25
27
|
postgresql
|
|
26
28
|
postgresql_makara
|
|
29
|
+
postgresql_proxy
|
|
27
30
|
postgis
|
|
28
31
|
makara_postgis
|
|
29
32
|
sqlite3
|
|
30
33
|
spatialite
|
|
31
34
|
seamless_database_pool
|
|
35
|
+
trilogy
|
|
36
|
+
janus_trilogy
|
|
32
37
|
).freeze
|
|
33
38
|
ADAPTERS.each do |adapter|
|
|
34
39
|
namespace :test do
|
data/activerecord-import.gemspec
CHANGED
|
@@ -10,6 +10,10 @@ Gem::Specification.new do |gem|
|
|
|
10
10
|
gem.homepage = "https://github.com/zdennis/activerecord-import"
|
|
11
11
|
gem.license = "MIT"
|
|
12
12
|
|
|
13
|
+
gem.metadata = {
|
|
14
|
+
"changelog_uri" => "https://github.com/zdennis/activerecord-import/blob/master/CHANGELOG.md"
|
|
15
|
+
}
|
|
16
|
+
|
|
13
17
|
gem.files = `git ls-files`.split($\)
|
|
14
18
|
gem.executables = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
|
|
15
19
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
data/benchmarks/benchmark.rb
CHANGED
|
@@ -41,11 +41,9 @@ require File.join(benchmark_dir, "../test/schema/generic_schema")
|
|
|
41
41
|
adapter_schema = File.join(benchmark_dir, "schema/#{options.adapter}_schema.rb")
|
|
42
42
|
require adapter_schema if File.exist?(adapter_schema)
|
|
43
43
|
|
|
44
|
-
Dir[File.dirname(__FILE__)
|
|
44
|
+
Dir["#{File.dirname(__FILE__)}/models/*.rb"].sort.each { |file| require file }
|
|
45
45
|
|
|
46
46
|
require File.join( benchmark_dir, 'lib', "#{options.adapter}_benchmark" )
|
|
47
|
-
|
|
48
|
-
table_types = nil
|
|
49
47
|
table_types = if options.benchmark_all_types
|
|
50
48
|
["all"]
|
|
51
49
|
else
|
|
@@ -53,8 +51,8 @@ else
|
|
|
53
51
|
end
|
|
54
52
|
|
|
55
53
|
letter = options.adapter[0].chr
|
|
56
|
-
clazz_str = letter.upcase + options.adapter[1
|
|
57
|
-
clazz = Object.const_get( clazz_str
|
|
54
|
+
clazz_str = letter.upcase + options.adapter[1..].downcase
|
|
55
|
+
clazz = Object.const_get( "#{clazz_str}Benchmark" )
|
|
58
56
|
|
|
59
57
|
benchmarks = []
|
|
60
58
|
options.number_of_objects.each do |num|
|