abstract_importer 1.5.4 → 1.7.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: ddf30730dc02843631c647c8f342623ebc46a87e
4
- data.tar.gz: 571852378a10de0e78df4952ad9241c0af25092d
2
+ SHA256:
3
+ metadata.gz: beb6af75a939202a48788cb3cbd6513c074ff514f1d8c7d104b32700ffdcfe63
4
+ data.tar.gz: 7ce453f1c101cb56a17642d5b0d21414db6cfd7de3221567f5bce75b4850e93f
5
5
  SHA512:
6
- metadata.gz: ac466e7b1f05a6074ae42d8e9dc188b641f8210caf95277e54830a4dd05a0c73a794392a8ee77dca07c5c600fb4ff47b9d5acc54f41c5f3c677c396db788ead9
7
- data.tar.gz: 3283e7ad8cb0b45d5364d917efdc28565755114f099113000b358411c1ec25e5c5fe3e3a75cf78b483b825bcf19cf6f1d4f655aca1ba8aaec5fa7420c434da45
6
+ metadata.gz: b17a2f043ce0a0a7fde5e6de5f1bea224d27142c7bfece25a3e890be4bfb0b4110d6413384949a8d3af4eeec19ad688a8e940556a55e65fa6ef9450468332a20
7
+ data.tar.gz: 70a4b060155c6f3104f57654340af8dc43816cddbd1a83c49295b647241e326f7e1edc163f7bf673a3d251cf7d02b7734c320cfceae85e3e277705676756f096
@@ -0,0 +1,29 @@
1
+ name: Tests
2
+ on: [push]
3
+
4
+ jobs:
5
+ ruby:
6
+ name: Tests
7
+ runs-on: ubuntu-latest
8
+ services:
9
+ postgres:
10
+ image: postgres:10.11
11
+ env:
12
+ POSTGRES_PASSWORD: password
13
+ POSTGRES_DB: abstract_importer_test
14
+ ports:
15
+ - 5432:5432
16
+ options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
17
+ steps:
18
+ - name: Checkout
19
+ uses: actions/checkout@v2
20
+ - name: Setup Ruby
21
+ uses: ruby/setup-ruby@v1
22
+ with:
23
+ ruby-version: 2.6
24
+ bundler-cache: true
25
+ - name: Run the Tests
26
+ env:
27
+ DATABASE_URL: postgres://postgres:password@localhost:5432/abstract_importer_test
28
+ run: bundle exec rake test
29
+
data/.ruby-version CHANGED
@@ -1 +1 @@
1
- 2.3.1
1
+ 2.6.3
data/CHANGELOG.md ADDED
@@ -0,0 +1,5 @@
1
+ ## v1.7.0 (2021 Nov 19)
2
+ * Updated to handle `insert_all` and `upsert_all` to `has_many through:` relations in Rails 6.1+
3
+
4
+ ## v1.6.0 (2019 Sept 23)
5
+ * BREAKING: Updated to use Rails 6 and its `insert_all` and `upsert_all` methods and arguments
data/README.md CHANGED
@@ -187,6 +187,23 @@ The following alternate strategies are built in:
187
187
 
188
188
  Replaces records that have already been imported rather than skipping them.
189
189
 
190
+ ##### insert
191
+
192
+ Bulk inserts records rather than creating them one-by-one as ActiveRecord objects, skipping those that have already been imported.
193
+
194
+ ##### upsert
195
+
196
+ Bulk inserts records, but updates records that have already been imported rather than skipping them.
197
+
198
+
199
+
200
+ ### Important Note on Upgrading from 1.5.x to 1.6.0
201
+
202
+ With the jump to 1.6.0, `abstract_importer` has dropped reliance upon the `activerecord-insert_many` gem in favor of taking advantage of Rails 6's built-in `insert_all` and `upsert_all` for the insert and upsert strategies. With the move from the gem to relying on Rails, the syntax to specify what constitutes a conflict/duplicate record has been updated to align more closely with that of Rails. In particular:
203
+
204
+ * Instead of specifying an `on_conflict: { do: :update }` clause, you should specify `on_duplicate: :update` to opt-in to upsert behavior.
205
+ * Rather than listing columns in `on_conflict: { columns: %i{legacy_id id} }`, you should use `unique_by: %i{legacy_id id}` to specify the index or columns by which a duplicate or conflict is defined. The option follows the behavior of Rails' `upsert_all`, so refer to the Rails 6 documentation for more information.
206
+
190
207
 
191
208
 
192
209
  ## Contributing
@@ -17,17 +17,16 @@ Gem::Specification.new do |spec|
17
17
  spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
18
18
  spec.require_paths = ["lib"]
19
19
 
20
- spec.add_dependency "activerecord", ">= 5.0"
21
- spec.add_dependency "activesupport", ">= 5.0"
22
- spec.add_dependency "activerecord-insert_many", ">= 0.4.3"
20
+ spec.add_dependency "activerecord", ">= 6.0"
21
+ spec.add_dependency "activesupport", ">= 6.0"
23
22
  spec.add_dependency "progressbar"
24
23
 
25
- spec.add_development_dependency "bundler", "~> 1.3"
24
+ spec.add_development_dependency "bundler"
26
25
  spec.add_development_dependency "minitest-reporters"
27
26
  spec.add_development_dependency "minitest-reporters-turn_reporter"
28
27
  spec.add_development_dependency "rake"
29
28
  spec.add_development_dependency "sqlite3"
30
- spec.add_development_dependency "pg", "~> 0.18"
29
+ spec.add_development_dependency "pg"
31
30
  spec.add_development_dependency "pry"
32
31
  spec.add_development_dependency "rr"
33
32
  spec.add_development_dependency "database_cleaner"
@@ -1,5 +1,4 @@
1
1
  require "abstract_importer/strategies/base"
2
- require "activerecord/insert_many"
3
2
 
4
3
  module AbstractImporter
5
4
  module Strategies
@@ -9,7 +8,8 @@ module AbstractImporter
9
8
  super
10
9
  @batch = []
11
10
  @batch_size = options.fetch(:batch_size, 250)
12
- @insert_options = options.slice(:on_conflict)
11
+ @bulk_operation = options[:on_duplicate] == :update ? :upsert_all : :insert_all
12
+ @insert_options = options.slice(:unique_by)
13
13
  @insert_options.merge!(returning: [:legacy_id, :id]) if remap_ids?
14
14
  end
15
15
 
@@ -52,7 +52,13 @@ module AbstractImporter
52
52
 
53
53
 
54
54
  def insert_batch(batch)
55
- result = collection.scope.insert_many(batch, @insert_options)
55
+ return if batch.empty?
56
+
57
+ scope = collection.scope
58
+ if scope.respond_to?(:proxy_association) && scope.proxy_association.reflection.through_reflection?
59
+ scope = scope.klass
60
+ end
61
+ result = scope.public_send(@bulk_operation, batch, @insert_options)
56
62
  add_batch_to_id_map(result) if remap_ids?
57
63
  end
58
64
 
@@ -66,13 +72,26 @@ module AbstractImporter
66
72
 
67
73
 
68
74
  def add_batch_to_id_map(result)
69
- map = result.each_with_object({}) do |attrs, map|
75
+ map = cast_result(result, collection.table_name).each_with_object({}) do |attrs, map|
70
76
  map[attrs.fetch("legacy_id")] = attrs.fetch("id")
71
77
  end
72
78
  id_map.merge! collection.table_name, map
73
79
  end
74
80
 
75
81
 
82
+ def cast_result(result, table_name)
83
+ types_by_column = result.columns.each_with_object({}) do |column_name, types|
84
+ types[column_name] = collection.scope.connection.lookup_cast_type_from_column(collection.scope.columns.find { |column| column.name == column_name })
85
+ end
86
+
87
+ result.to_a.map { |row|
88
+ Hash[row.map { |column_name, value|
89
+ [ column_name, types_by_column[column_name].deserialize(value) ]
90
+ }]
91
+ }
92
+ end
93
+
94
+
76
95
  end
77
96
  end
78
97
  end
@@ -1,5 +1,4 @@
1
1
  require "abstract_importer/strategies/insert_strategy"
2
- require "activerecord/insert_many"
3
2
 
4
3
  module AbstractImporter
5
4
  module Strategies
@@ -7,7 +6,8 @@ module AbstractImporter
7
6
 
8
7
  def initialize(collection, options={})
9
8
  super
10
- @insert_options.merge!(on_conflict: { column: remap_ids? ? (association_attrs.keys + [:legacy_id]) : :id, do: :update })
9
+ @bulk_operation = :upsert_all
10
+ @insert_options.reverse_merge!(unique_by: remap_ids? ? (association_attrs.keys + %i{legacy_id}) : :id)
11
11
  end
12
12
 
13
13
  # We won't skip any records for already being imported
@@ -1,3 +1,3 @@
1
1
  module AbstractImporter
2
- VERSION = "1.5.4"
2
+ VERSION = "1.7.0"
3
3
  end
@@ -147,7 +147,7 @@ class DefaultStrategyTest < ActiveSupport::TestCase
147
147
  import!
148
148
  assert_equal 2, account.students.map(&:pet).compact.count, "Expected two students to still be linked to their pets upon import"
149
149
  assert_kind_of Owl, account.students.find_by_name("Harry Potter").pet, "Expected Harry's pet to be an Owl"
150
- assert_kind_of Cat, account.students.find_by_name("Hermione Granger").pet, "Expected Harry's pet to be a Cat"
150
+ assert_kind_of Cat, account.students.find_by_name("Hermione Granger").pet, "Expected Hermione's pet to be a Cat"
151
151
  end
152
152
  end
153
153
 
@@ -4,7 +4,7 @@ require "test_helper"
4
4
  class InsertStrategyTest < ActiveSupport::TestCase
5
5
 
6
6
  setup do
7
- options.merge!(strategy: {students: :insert})
7
+ options.merge!(strategy: {students: :insert, perils: :insert})
8
8
  end
9
9
 
10
10
 
@@ -17,7 +17,7 @@ class InsertStrategyTest < ActiveSupport::TestCase
17
17
  end
18
18
 
19
19
  should "import the records in batches" do
20
- mock.proxy(Student).insert_many(satisfy { |arg| arg.length == 3 }, anything)
20
+ mock.proxy(Student).insert_all(satisfy { |arg| arg.length == 3 }, anything)
21
21
  import!
22
22
  assert_equal [456, 457, 458], account.students.pluck(:legacy_id)
23
23
  end
@@ -54,6 +54,22 @@ class InsertStrategyTest < ActiveSupport::TestCase
54
54
  end
55
55
  end
56
56
 
57
+ context "With an empty data source" do
58
+ setup do
59
+ plan do |import|
60
+ import.students
61
+ end
62
+ @data_source = OpenStruct.new
63
+ @data_source.students = []
64
+ end
65
+
66
+ should "still be able to import" do
67
+ assert_nothing_raised do
68
+ import!
69
+ end
70
+ end
71
+ end
72
+
57
73
  context "When records already exist" do
58
74
  setup do
59
75
  plan do |import|
@@ -83,6 +99,20 @@ class InsertStrategyTest < ActiveSupport::TestCase
83
99
  end
84
100
  end
85
101
 
102
+ context "When importing a has_many through: relationship" do
103
+ setup do
104
+ plan do |import|
105
+ import.locations
106
+ import.perils
107
+ end
108
+ end
109
+
110
+ should "handle has_many, through: relations" do
111
+ import!
112
+ assert_equal 1, account.perils.count
113
+ end
114
+ end
115
+
86
116
 
87
117
 
88
118
  context "Given an ID generator" do
@@ -58,5 +58,11 @@ class MockDataSource
58
58
  end
59
59
  end
60
60
 
61
+ def perils
62
+ Enumerator.new do |e|
63
+ e.yield id: 801, name: "Dementors", location_id: 6
64
+ end
65
+ end
66
+
61
67
 
62
68
  end
@@ -17,6 +17,7 @@ class Parent < ActiveRecord::Base
17
17
  end
18
18
 
19
19
  class Location < ActiveRecord::Base
20
+ has_many :perils
20
21
  validates :slug, format: {with: /\A[a-z0-9\-]+\z/}
21
22
  end
22
23
 
@@ -37,6 +38,8 @@ class Account < ActiveRecord::Base
37
38
  has_many :locations
38
39
  has_many :cats
39
40
  has_many :owls
41
+
42
+ has_many :perils, through: :locations
40
43
  end
41
44
 
42
45
  class Cat < ActiveRecord::Base
@@ -52,3 +55,7 @@ end
52
55
  class Ability < ActiveRecord::Base
53
56
  belongs_to :pet, inverse_of: :abilities, polymorphic: true
54
57
  end
58
+
59
+ class Peril < ActiveRecord::Base
60
+ belongs_to :location
61
+ end
@@ -72,4 +72,11 @@ ActiveRecord::Schema.define(:version => 1) do
72
72
  t.index "legacy_id", :unique => true
73
73
  end
74
74
 
75
+ create_table "perils", :force => true do |t|
76
+ t.string "name"
77
+ t.integer "location_id"
78
+ t.integer "legacy_id"
79
+ t.index "legacy_id", :unique => true
80
+ end
81
+
75
82
  end
data/test/test_helper.rb CHANGED
@@ -22,11 +22,15 @@ require "minitest/autorun"
22
22
 
23
23
  system "psql -c 'create database abstract_importer_test'"
24
24
 
25
- ActiveRecord::Base.establish_connection(
26
- adapter: "postgresql",
27
- host: "localhost",
28
- database: "abstract_importer_test",
29
- verbosity: "quiet")
25
+ if ENV["DATABASE_URL"]
26
+ ActiveRecord::Base.establish_connection(ENV["DATABASE_URL"])
27
+ else
28
+ ActiveRecord::Base.establish_connection(
29
+ adapter: "postgresql",
30
+ host: "localhost",
31
+ database: "abstract_importer_test",
32
+ verbosity: "quiet")
33
+ end
30
34
 
31
35
  load File.join(File.dirname(__FILE__), "support", "schema.rb")
32
36
 
@@ -17,7 +17,7 @@ class UpsertStrategyTest < ActiveSupport::TestCase
17
17
  end
18
18
 
19
19
  should "import the records in batches" do
20
- mock.proxy(Student).insert_many(satisfy { |arg| arg.length == 3 }, anything)
20
+ mock.proxy(Student).upsert_all(satisfy { |arg| arg.length == 3 }, anything)
21
21
  import!
22
22
  assert_equal [456, 457, 458], account.students.pluck(:legacy_id)
23
23
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: abstract_importer
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.5.4
4
+ version: 1.7.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Bob Lail
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-09-11 00:00:00.000000000 Z
11
+ date: 2021-11-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activerecord
@@ -16,42 +16,28 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: '5.0'
19
+ version: '6.0'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - ">="
25
25
  - !ruby/object:Gem::Version
26
- version: '5.0'
26
+ version: '6.0'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: activesupport
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - ">="
32
32
  - !ruby/object:Gem::Version
33
- version: '5.0'
33
+ version: '6.0'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
- version: '5.0'
41
- - !ruby/object:Gem::Dependency
42
- name: activerecord-insert_many
43
- requirement: !ruby/object:Gem::Requirement
44
- requirements:
45
- - - ">="
46
- - !ruby/object:Gem::Version
47
- version: 0.4.3
48
- type: :runtime
49
- prerelease: false
50
- version_requirements: !ruby/object:Gem::Requirement
51
- requirements:
52
- - - ">="
53
- - !ruby/object:Gem::Version
54
- version: 0.4.3
40
+ version: '6.0'
55
41
  - !ruby/object:Gem::Dependency
56
42
  name: progressbar
57
43
  requirement: !ruby/object:Gem::Requirement
@@ -70,16 +56,16 @@ dependencies:
70
56
  name: bundler
71
57
  requirement: !ruby/object:Gem::Requirement
72
58
  requirements:
73
- - - "~>"
59
+ - - ">="
74
60
  - !ruby/object:Gem::Version
75
- version: '1.3'
61
+ version: '0'
76
62
  type: :development
77
63
  prerelease: false
78
64
  version_requirements: !ruby/object:Gem::Requirement
79
65
  requirements:
80
- - - "~>"
66
+ - - ">="
81
67
  - !ruby/object:Gem::Version
82
- version: '1.3'
68
+ version: '0'
83
69
  - !ruby/object:Gem::Dependency
84
70
  name: minitest-reporters
85
71
  requirement: !ruby/object:Gem::Requirement
@@ -140,16 +126,16 @@ dependencies:
140
126
  name: pg
141
127
  requirement: !ruby/object:Gem::Requirement
142
128
  requirements:
143
- - - "~>"
129
+ - - ">="
144
130
  - !ruby/object:Gem::Version
145
- version: '0.18'
131
+ version: '0'
146
132
  type: :development
147
133
  prerelease: false
148
134
  version_requirements: !ruby/object:Gem::Requirement
149
135
  requirements:
150
- - - "~>"
136
+ - - ">="
151
137
  - !ruby/object:Gem::Version
152
- version: '0.18'
138
+ version: '0'
153
139
  - !ruby/object:Gem::Dependency
154
140
  name: pry
155
141
  requirement: !ruby/object:Gem::Requirement
@@ -227,9 +213,10 @@ executables: []
227
213
  extensions: []
228
214
  extra_rdoc_files: []
229
215
  files:
216
+ - ".github/workflows/ci.yml"
230
217
  - ".gitignore"
231
218
  - ".ruby-version"
232
- - ".travis.yml"
219
+ - CHANGELOG.md
233
220
  - Gemfile
234
221
  - LICENSE.txt
235
222
  - README.md
@@ -288,8 +275,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
288
275
  - !ruby/object:Gem::Version
289
276
  version: '0'
290
277
  requirements: []
291
- rubyforge_project:
292
- rubygems_version: 2.6.11
278
+ rubygems_version: 3.0.3
293
279
  signing_key:
294
280
  specification_version: 4
295
281
  summary: Provides services for the mass-import of complex relational data
data/.travis.yml DELETED
@@ -1,17 +0,0 @@
1
- language: ruby
2
- rvm:
3
- - 2.3.1
4
-
5
- # Use Postgres 9.5
6
- # https://www.brandur.org/fragments/postgres-95-travis
7
- dist: trusty
8
- sudo: required
9
- addons:
10
- postgresql: "9.5"
11
-
12
- before_install: gem update bundler
13
- script: bundle exec rake test
14
-
15
- # To stop Travis from running tests for a new commit,
16
- # add the following to your commit message: [ci skip]
17
- # You should add this when you edit documentation or comments, etc.