abstract_importer 1.4.0 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.ruby-version +1 -1
- data/abstract_importer.gemspec +1 -1
- data/lib/abstract_importer/base.rb +23 -17
- data/lib/abstract_importer/collection.rb +11 -5
- data/lib/abstract_importer/strategies/insert_strategy.rb +2 -1
- data/lib/abstract_importer/strategies/upsert_strategy.rb +4 -9
- data/lib/abstract_importer/version.rb +1 -1
- data/test/insert_strategy_test.rb +1 -1
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f9366e2d43d5040adc38eca851c665c22a8f15ee
|
4
|
+
data.tar.gz: 441e4ea070305989b22e5ba88d9111782012dd5f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4b886e1366ec2c04c3291a25e9b0c0f6ce3b970197aefbd5598a0894a21be79e598b23396969edde8f7e83291bf60ef36cd1f01e4e50979217fd9e729daf0388
|
7
|
+
data.tar.gz: 0bfd28f4e398e928d49fa6dbcfc3f5dc31f7d30517b35acdd5dbb2e57f38b3e1021b27a8b338544b11da79c4abb450835faa4430dff9470c2935de187ad0d6e9
|
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
2.3.
|
1
|
+
2.3.1
|
data/abstract_importer.gemspec
CHANGED
@@ -9,7 +9,7 @@ Gem::Specification.new do |spec|
|
|
9
9
|
spec.authors = ["Bob Lail"]
|
10
10
|
spec.email = ["bob.lail@cph.org"]
|
11
11
|
spec.summary = %q{Provides services for the mass-import of complex relational data}
|
12
|
-
spec.homepage = "https://github.com/
|
12
|
+
spec.homepage = "https://github.com/cph/abstract_importer"
|
13
13
|
spec.license = "MIT"
|
14
14
|
|
15
15
|
spec.files = `git ls-files`.split($/)
|
@@ -21,7 +21,11 @@ module AbstractImporter
|
|
21
21
|
@dependencies = dependencies
|
22
22
|
end
|
23
23
|
|
24
|
-
|
24
|
+
def dependencies
|
25
|
+
@dependencies ||= []
|
26
|
+
end
|
27
|
+
|
28
|
+
attr_reader :import_plan
|
25
29
|
end
|
26
30
|
|
27
31
|
|
@@ -185,9 +189,21 @@ module AbstractImporter
|
|
185
189
|
|
186
190
|
|
187
191
|
|
192
|
+
protected
|
193
|
+
|
194
|
+
def scope_for(collection_name)
|
195
|
+
parent.public_send(collection_name)
|
196
|
+
end
|
188
197
|
|
189
198
|
private
|
190
199
|
|
200
|
+
def has_scope_for?(collection_name)
|
201
|
+
scope_for(collection_name)
|
202
|
+
true
|
203
|
+
rescue NoMethodError
|
204
|
+
false
|
205
|
+
end
|
206
|
+
|
191
207
|
def verify_source!
|
192
208
|
import_plan.keys.each do |collection|
|
193
209
|
next if source.respond_to?(collection)
|
@@ -199,14 +215,14 @@ module AbstractImporter
|
|
199
215
|
|
200
216
|
def verify_parent!
|
201
217
|
import_plan.keys.each do |collection|
|
202
|
-
next if
|
218
|
+
next if has_scope_for?(collection)
|
203
219
|
|
204
220
|
raise "#{parent.class} does not have a collection named `#{collection}`; " <<
|
205
221
|
"but #{self.class} plans to import records with that name"
|
206
222
|
end
|
207
223
|
|
208
|
-
|
209
|
-
next if
|
224
|
+
self.class.dependencies.each do |collection|
|
225
|
+
next if has_scope_for?(collection)
|
210
226
|
|
211
227
|
raise "#{parent.class} does not have a collection named `#{collection}`; " <<
|
212
228
|
"but #{self.class} declares it as a dependency"
|
@@ -215,26 +231,16 @@ module AbstractImporter
|
|
215
231
|
|
216
232
|
def instantiate_collections!
|
217
233
|
@collections = import_plan.map do |name, block|
|
218
|
-
reflection = parent.class.reflect_on_association(name)
|
219
|
-
model = reflection.klass
|
220
|
-
table_name = model.table_name
|
221
|
-
scope = parent.public_send(name)
|
222
|
-
|
223
234
|
options = ImportOptions.new
|
224
235
|
instance_exec(options, &block) if block
|
225
236
|
|
226
|
-
Collection.new(name,
|
237
|
+
Collection.new(name, scope_for(name), options)
|
227
238
|
end
|
228
239
|
end
|
229
240
|
|
230
241
|
def dependencies
|
231
|
-
@dependencies ||=
|
232
|
-
|
233
|
-
model = reflection.klass
|
234
|
-
table_name = model.table_name
|
235
|
-
scope = parent.public_send(name)
|
236
|
-
|
237
|
-
Collection.new(name, model, table_name, scope, nil)
|
242
|
+
@dependencies ||= self.class.dependencies.map do |name|
|
243
|
+
Collection.new(name, scope_for(name))
|
238
244
|
end
|
239
245
|
end
|
240
246
|
|
@@ -1,6 +1,10 @@
|
|
1
1
|
module AbstractImporter
|
2
2
|
class Collection < Struct.new(:name, :model, :table_name, :scope, :options)
|
3
3
|
|
4
|
+
def initialize(name, scope, options=nil)
|
5
|
+
super name, scope.model, scope.table_name, scope, options
|
6
|
+
end
|
7
|
+
|
4
8
|
def association_attrs
|
5
9
|
return @association_attrs if defined?(@association_attrs)
|
6
10
|
|
@@ -9,11 +13,13 @@ module AbstractImporter
|
|
9
13
|
# can call `Person.new(__.merge(tenant_id: id))`.
|
10
14
|
@association_attrs = {}
|
11
15
|
association = scope.instance_variable_get(:@association)
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
16
|
+
if association
|
17
|
+
unless association.is_a?(ActiveRecord::Associations::HasManyThroughAssociation)
|
18
|
+
@association_attrs.merge!(association.reflection.foreign_key.to_sym => association.owner.id)
|
19
|
+
end
|
20
|
+
if association.reflection.inverse_of && association.reflection.inverse_of.polymorphic?
|
21
|
+
@association_attrs.merge!(association.reflection.inverse_of.foreign_type.to_sym => association.owner.class.name)
|
22
|
+
end
|
17
23
|
end
|
18
24
|
@association_attrs.freeze
|
19
25
|
end
|
@@ -9,6 +9,7 @@ module AbstractImporter
|
|
9
9
|
super
|
10
10
|
@batch = []
|
11
11
|
@batch_size = options.fetch(:batch_size, 250)
|
12
|
+
@insert_options = options.slice(:on_conflict)
|
12
13
|
end
|
13
14
|
|
14
15
|
|
@@ -49,7 +50,7 @@ module AbstractImporter
|
|
49
50
|
|
50
51
|
|
51
52
|
def insert_batch(batch)
|
52
|
-
collection.scope.insert_many(batch)
|
53
|
+
collection.scope.insert_many(batch, @insert_options)
|
53
54
|
end
|
54
55
|
|
55
56
|
|
@@ -5,21 +5,16 @@ module AbstractImporter
|
|
5
5
|
module Strategies
|
6
6
|
class UpsertStrategy < InsertStrategy
|
7
7
|
|
8
|
+
def initialize(collection, options={})
|
9
|
+
super
|
10
|
+
@insert_options.merge!(on_conflict: { column: remap_ids? ? :legacy_id : :id, do: :update })
|
11
|
+
end
|
8
12
|
|
9
13
|
# We won't skip any records for already being imported
|
10
14
|
def already_imported?(hash)
|
11
15
|
false
|
12
16
|
end
|
13
17
|
|
14
|
-
|
15
|
-
def insert_batch(batch)
|
16
|
-
collection.scope.insert_many(batch, on_conflict: {
|
17
|
-
column: remap_ids? ? :legacy_id : :id,
|
18
|
-
do: :update
|
19
|
-
})
|
20
|
-
end
|
21
|
-
|
22
|
-
|
23
18
|
end
|
24
19
|
end
|
25
20
|
end
|
@@ -17,7 +17,7 @@ class InsertStrategyTest < ActiveSupport::TestCase
|
|
17
17
|
end
|
18
18
|
|
19
19
|
should "import the records in batches" do
|
20
|
-
mock.proxy(Student).insert_many(satisfy { |arg| arg.length == 3 })
|
20
|
+
mock.proxy(Student).insert_many(satisfy { |arg| arg.length == 3 }, {})
|
21
21
|
import!
|
22
22
|
assert_equal [456, 457, 458], account.students.pluck(:legacy_id)
|
23
23
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: abstract_importer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.4.
|
4
|
+
version: 1.4.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Bob Lail
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-04-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -267,7 +267,7 @@ files:
|
|
267
267
|
- test/support/schema.rb
|
268
268
|
- test/test_helper.rb
|
269
269
|
- test/upsert_strategy_test.rb
|
270
|
-
homepage: https://github.com/
|
270
|
+
homepage: https://github.com/cph/abstract_importer
|
271
271
|
licenses:
|
272
272
|
- MIT
|
273
273
|
metadata: {}
|
@@ -287,7 +287,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
287
287
|
version: '0'
|
288
288
|
requirements: []
|
289
289
|
rubyforge_project:
|
290
|
-
rubygems_version: 2.
|
290
|
+
rubygems_version: 2.6.11
|
291
291
|
signing_key:
|
292
292
|
specification_version: 4
|
293
293
|
summary: Provides services for the mass-import of complex relational data
|