duty_free 1.0.7 → 1.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/duty_free.rb +200 -50
- data/lib/duty_free/column.rb +2 -2
- data/lib/duty_free/extensions.rb +586 -439
- data/lib/duty_free/suggest_template.rb +10 -11
- data/lib/duty_free/util.rb +129 -11
- data/lib/duty_free/version_number.rb +1 -1
- metadata +11 -25
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 714b772aa09eb15409cd7c58d4ccb64f967caf41f2a13e7813e90cc94d7eb48f
|
4
|
+
data.tar.gz: 750dfe36b729979a0356f14f8588568822ae22911c142d294358231504daa78e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9d77a228b4e4990dad39882a4e67d1009f3c16114bc0e9b24280e1f8e9437c7fc55a8884eb37e8f6c15898958833c7f79bb7984d721f4dbd1fce7311ca6421d0
|
7
|
+
data.tar.gz: 1d35f3197ab500a6210eecc561190c02a04d87e3d4b2634a9a62695488d4e2ed0825f7240836aa9d32fe8ce01e06be38ddccda1955ca3d3669fd7ad18e8335f2
|
data/lib/duty_free.rb
CHANGED
@@ -21,8 +21,8 @@ if ActiveRecord.version < ::Gem::Version.new('5.0') &&
|
|
21
21
|
end
|
22
22
|
end
|
23
23
|
|
24
|
-
# Allow
|
25
|
-
# when ActiveSupport tries to smarten up Numeric by messing with Fixnum and Bignum at the end of:
|
24
|
+
# Allow ActiveRecord 4.0 and 4.1 to work with newer Ruby (>= 2.4) by avoiding a "stack level too deep"
|
25
|
+
# error when ActiveSupport tries to smarten up Numeric by messing with Fixnum and Bignum at the end of:
|
26
26
|
# activesupport-4.0.13/lib/active_support/core_ext/numeric/conversions.rb
|
27
27
|
if ActiveRecord.version < ::Gem::Version.new('4.2') &&
|
28
28
|
ActiveRecord.version > ::Gem::Version.new('3.2') &&
|
@@ -33,7 +33,7 @@ if ActiveRecord.version < ::Gem::Version.new('4.2') &&
|
|
33
33
|
Numeric.const_set('Bignum', OurBignum)
|
34
34
|
end
|
35
35
|
|
36
|
-
# Allow
|
36
|
+
# Allow ActiveRecord < 3.2 to run with newer versions of Psych gem
|
37
37
|
if BigDecimal.respond_to?(:yaml_tag) && !BigDecimal.respond_to?(:yaml_as)
|
38
38
|
class BigDecimal
|
39
39
|
class <<self
|
@@ -42,6 +42,28 @@ if BigDecimal.respond_to?(:yaml_tag) && !BigDecimal.respond_to?(:yaml_as)
|
|
42
42
|
end
|
43
43
|
end
|
44
44
|
|
45
|
+
require 'duty_free/util'
|
46
|
+
|
47
|
+
# Allow ActiveRecord < 3.2 to work with Ruby 2.7 and later
|
48
|
+
if ActiveRecord.version < ::Gem::Version.new('3.2') &&
|
49
|
+
::Gem::Version.new(RUBY_VERSION) >= ::Gem::Version.new('2.7')
|
50
|
+
# Remove circular reference for "now"
|
51
|
+
::DutyFree::Util._patch_require(
|
52
|
+
'active_support/values/time_zone.rb', '/activesupport',
|
53
|
+
' def parse(str, now=now)',
|
54
|
+
' def parse(str, now=now())'
|
55
|
+
)
|
56
|
+
# Remove circular reference for "reflection" for ActiveRecord 3.1
|
57
|
+
if ActiveRecord.version >= ::Gem::Version.new('3.1')
|
58
|
+
::DutyFree::Util._patch_require(
|
59
|
+
'active_record/associations/has_many_association.rb', '/activerecord',
|
60
|
+
'reflection = reflection)',
|
61
|
+
'reflection = reflection())',
|
62
|
+
:HasManyAssociation # Make sure the path for this guy is available to be autoloaded
|
63
|
+
)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
45
67
|
require 'active_record'
|
46
68
|
|
47
69
|
require 'duty_free/config'
|
@@ -107,14 +129,19 @@ end
|
|
107
129
|
# Major compatibility fixes for ActiveRecord < 4.2
|
108
130
|
# ================================================
|
109
131
|
ActiveSupport.on_load(:active_record) do
|
110
|
-
# Rails < 4.0 cannot do #find_by, or do #pluck on multiple columns, so here are the patches:
|
132
|
+
# Rails < 4.0 cannot do #find_by, #find_or_create_by, or do #pluck on multiple columns, so here are the patches:
|
111
133
|
if ActiveRecord.version < ::Gem::Version.new('4.0')
|
112
134
|
module ActiveRecord
|
113
|
-
|
135
|
+
# Normally find_by is in FinderMethods, which older AR doesn't have
|
136
|
+
module Calculations
|
114
137
|
def find_by(*args)
|
115
138
|
where(*args).limit(1).to_a.first
|
116
139
|
end
|
117
140
|
|
141
|
+
def find_or_create_by(attributes, &block)
|
142
|
+
find_by(attributes) || create(attributes, &block)
|
143
|
+
end
|
144
|
+
|
118
145
|
def pluck(*column_names)
|
119
146
|
column_names.map! do |column_name|
|
120
147
|
if column_name.is_a?(Symbol) && self.column_names.include?(column_name.to_s)
|
@@ -174,7 +201,7 @@ ActiveSupport.on_load(:active_record) do
|
|
174
201
|
unless Base.is_a?(Calculations)
|
175
202
|
class Base
|
176
203
|
class << self
|
177
|
-
delegate :pluck, :find_by, to: :scoped
|
204
|
+
delegate :pluck, :find_by, :find_or_create_by, to: :scoped
|
178
205
|
end
|
179
206
|
end
|
180
207
|
end
|
@@ -205,50 +232,105 @@ ActiveSupport.on_load(:active_record) do
|
|
205
232
|
end
|
206
233
|
end
|
207
234
|
end
|
208
|
-
end
|
209
|
-
end
|
210
235
|
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
236
|
+
# ActiveRecord 3.1 and 3.2 didn't try to bring in &block for the .extending() convenience thing
|
237
|
+
# that smartens up scopes, and Ruby 2.7 complained loudly about just doing the magical "Proc.new"
|
238
|
+
# that historically would just capture the incoming block.
|
239
|
+
module QueryMethods
|
240
|
+
unless instance_method(:extending).parameters.include?([:block, :block])
|
241
|
+
# These first two lines used to be:
|
242
|
+
# def extending(*modules)
|
243
|
+
# modules << Module.new(&Proc.new) if block_given?
|
244
|
+
|
245
|
+
def extending(*modules, &block)
|
246
|
+
modules << Module.new(&block) if block_given?
|
247
|
+
|
248
|
+
return self if modules.empty?
|
220
249
|
|
221
|
-
|
222
|
-
|
250
|
+
relation = clone
|
251
|
+
relation.send(:apply_modules, modules.flatten)
|
252
|
+
relation
|
223
253
|
end
|
254
|
+
end
|
255
|
+
end
|
224
256
|
|
225
|
-
|
226
|
-
|
257
|
+
# Same kind of thing for ActiveRecord::Scoping::Default#default_scope
|
258
|
+
module Scoping
|
259
|
+
module Default
|
260
|
+
module ClassMethods
|
261
|
+
if instance_methods.include?(:default_scope) &&
|
262
|
+
!instance_method(:default_scope).parameters.include?([:block, :block])
|
263
|
+
# Fix for AR 3.2-5.1
|
264
|
+
def default_scope(scope = nil, &block)
|
265
|
+
scope = block if block_given?
|
266
|
+
|
267
|
+
if scope.is_a?(Relation) || !scope.respond_to?(:call)
|
268
|
+
raise ArgumentError,
|
269
|
+
'Support for calling #default_scope without a block is removed. For example instead ' \
|
270
|
+
"of `default_scope where(color: 'red')`, please use " \
|
271
|
+
"`default_scope { where(color: 'red') }`. (Alternatively you can just redefine " \
|
272
|
+
'self.default_scope.)'
|
273
|
+
end
|
227
274
|
|
228
|
-
|
229
|
-
unless private_instance_methods.include?(:literal)
|
230
|
-
def literal(obj)
|
231
|
-
obj
|
275
|
+
self.default_scopes += [scope]
|
232
276
|
end
|
233
277
|
end
|
234
|
-
alias visit_Integer literal
|
235
278
|
end
|
236
279
|
end
|
237
280
|
end
|
238
281
|
end
|
239
282
|
end
|
240
283
|
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
284
|
+
# Rails < 4.2 is not innately compatible with Ruby 2.4 and later, and comes up with:
|
285
|
+
# "TypeError: Cannot visit Integer" unless we patch like this:
|
286
|
+
if ::Gem::Version.new(RUBY_VERSION) >= ::Gem::Version.new('2.4') &&
|
287
|
+
Arel::Visitors.const_defined?('DepthFirst') &&
|
288
|
+
!Arel::Visitors::DepthFirst.private_instance_methods.include?(:visit_Integer)
|
289
|
+
module Arel
|
290
|
+
module Visitors
|
291
|
+
class DepthFirst < Visitor
|
292
|
+
alias visit_Integer terminal
|
293
|
+
end
|
294
|
+
|
295
|
+
class Dot < Visitor
|
296
|
+
alias visit_Integer visit_String
|
297
|
+
end
|
298
|
+
|
299
|
+
class ToSql < Visitor
|
300
|
+
private
|
301
|
+
|
302
|
+
# ActiveRecord before v3.2 uses Arel < 3.x, which does not have Arel#literal.
|
303
|
+
unless private_instance_methods.include?(:literal)
|
304
|
+
def literal(obj)
|
305
|
+
obj
|
306
|
+
end
|
307
|
+
end
|
308
|
+
alias visit_Integer literal
|
309
|
+
end
|
310
|
+
end
|
311
|
+
end
|
246
312
|
end
|
247
313
|
|
248
314
|
unless DateTime.instance_methods.include?(:nsec)
|
249
315
|
class DateTime < Date
|
250
316
|
def nsec
|
251
|
-
(sec_fraction *
|
317
|
+
(sec_fraction * 1_000_000_000).to_i
|
318
|
+
end
|
319
|
+
end
|
320
|
+
end
|
321
|
+
|
322
|
+
# First part of arel_table_type stuff:
|
323
|
+
# ------------------------------------
|
324
|
+
# (more found below)
|
325
|
+
if ActiveRecord.version < ::Gem::Version.new('5.0')
|
326
|
+
# Used by Util#_arel_table_type
|
327
|
+
module ActiveRecord
|
328
|
+
class Base
|
329
|
+
def self.arel_table
|
330
|
+
@arel_table ||= Arel::Table.new(table_name, arel_engine).tap do |x|
|
331
|
+
x.instance_variable_set(:@_arel_table_type, self)
|
332
|
+
end
|
333
|
+
end
|
252
334
|
end
|
253
335
|
end
|
254
336
|
end
|
@@ -256,21 +338,89 @@ ActiveSupport.on_load(:active_record) do
|
|
256
338
|
include ::DutyFree::Extensions
|
257
339
|
end
|
258
340
|
|
259
|
-
#
|
260
|
-
|
261
|
-
#
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
#
|
268
|
-
#
|
269
|
-
|
270
|
-
#
|
271
|
-
|
272
|
-
#
|
273
|
-
#
|
274
|
-
|
275
|
-
|
276
|
-
#
|
341
|
+
# Do this earlier because stuff here gets mixed into JoinDependency::JoinAssociation and AssociationScope
|
342
|
+
if ActiveRecord.version < ::Gem::Version.new('5.0') && Object.const_defined?('PG::Connection')
|
343
|
+
# Avoid pg gem deprecation warning: "You should use PG::Connection, PG::Result, and PG::Error instead"
|
344
|
+
PGconn = PG::Connection
|
345
|
+
PGresult = PG::Result
|
346
|
+
PGError = PG::Error
|
347
|
+
end
|
348
|
+
|
349
|
+
# More arel_table_type stuff:
|
350
|
+
# ---------------------------
|
351
|
+
if ActiveRecord.version < ::Gem::Version.new('5.2')
|
352
|
+
# Specifically for AR 3.1 and 3.2 to avoid: "undefined method `delegate' for ActiveRecord::Reflection::ThroughReflection:Class"
|
353
|
+
require 'active_support/core_ext/module/delegation' if ActiveRecord.version < ::Gem::Version.new('4.0')
|
354
|
+
# Used by Util#_arel_table_type
|
355
|
+
# rubocop:disable Style/CommentedKeyword
|
356
|
+
module ActiveRecord
|
357
|
+
module Reflection
|
358
|
+
# AR < 4.0 doesn't know about join_table and derive_join_table
|
359
|
+
unless AssociationReflection.instance_methods.include?(:join_table)
|
360
|
+
class AssociationReflection < MacroReflection
|
361
|
+
def join_table
|
362
|
+
@join_table ||= options[:join_table] || derive_join_table
|
363
|
+
end
|
364
|
+
|
365
|
+
private
|
366
|
+
|
367
|
+
def derive_join_table
|
368
|
+
[active_record.table_name, klass.table_name].sort.join("\0").gsub(/^(.*[._])(.+)\0\1(.+)/, '\1\2_\3').gsub("\0", '_')
|
369
|
+
end
|
370
|
+
end
|
371
|
+
end
|
372
|
+
end
|
373
|
+
|
374
|
+
module Associations
|
375
|
+
# Specific to AR 4.2 - 5.1:
|
376
|
+
if Associations.const_defined?('JoinDependency') && JoinDependency.private_instance_methods.include?(:table_aliases_for)
|
377
|
+
class JoinDependency
|
378
|
+
private
|
379
|
+
|
380
|
+
if ActiveRecord.version < ::Gem::Version.new('5.1') # 4.2 or 5.0
|
381
|
+
def table_aliases_for(parent, node)
|
382
|
+
node.reflection.chain.map do |reflection|
|
383
|
+
alias_tracker.aliased_table_for(
|
384
|
+
reflection.table_name,
|
385
|
+
table_alias_for(reflection, parent, reflection != node.reflection)
|
386
|
+
).tap do |x|
|
387
|
+
# %%% Specific only to Rails 4.2 (and maybe 4.1?)
|
388
|
+
x = x.left if x.is_a?(Arel::Nodes::TableAlias)
|
389
|
+
y = reflection.chain.find { |c| c.table_name == x.name }
|
390
|
+
x.instance_variable_set(:@_arel_table_type, y.klass)
|
391
|
+
end
|
392
|
+
end
|
393
|
+
end
|
394
|
+
end
|
395
|
+
end
|
396
|
+
elsif Associations.const_defined?('JoinHelper') && JoinHelper.private_instance_methods.include?(:construct_tables)
|
397
|
+
module JoinHelper
|
398
|
+
private
|
399
|
+
|
400
|
+
# AR > 3.0 and < 4.2 (%%% maybe only < 4.1?) uses construct_tables like this:
|
401
|
+
def construct_tables
|
402
|
+
tables = []
|
403
|
+
chain.each do |reflection|
|
404
|
+
tables << alias_tracker.aliased_table_for(
|
405
|
+
table_name_for(reflection),
|
406
|
+
table_alias_for(reflection, reflection != self.reflection)
|
407
|
+
).tap do |x|
|
408
|
+
x = x.left if x.is_a?(Arel::Nodes::TableAlias)
|
409
|
+
x.instance_variable_set(:@_arel_table_type, reflection.chain.find { |c| c.table_name == x.name }.klass)
|
410
|
+
end
|
411
|
+
|
412
|
+
next unless reflection.source_macro == :has_and_belongs_to_many
|
413
|
+
|
414
|
+
tables << alias_tracker.aliased_table_for(
|
415
|
+
(reflection.source_reflection || reflection).join_table,
|
416
|
+
table_alias_for(reflection, true)
|
417
|
+
)
|
418
|
+
end
|
419
|
+
tables
|
420
|
+
end
|
421
|
+
end
|
422
|
+
end
|
423
|
+
end
|
424
|
+
end # module ActiveRecord
|
425
|
+
# rubocop:enable Style/CommentedKeyword
|
426
|
+
end
|
data/lib/duty_free/column.rb
CHANGED
@@ -36,7 +36,7 @@ module DutyFree
|
|
36
36
|
end
|
37
37
|
|
38
38
|
def titleize
|
39
|
-
@
|
39
|
+
@titleize ||= to_sym.titleize
|
40
40
|
end
|
41
41
|
|
42
42
|
def path
|
@@ -45,7 +45,7 @@ module DutyFree
|
|
45
45
|
|
46
46
|
# The snake-cased column name to be used for building the full list of template_columns
|
47
47
|
def to_sym
|
48
|
-
@
|
48
|
+
@to_sym ||= ::DutyFree::Util._prefix_join(
|
49
49
|
[pre_prefix, prefix, ::DutyFree::Util._clean_name(name, import_template_as)],
|
50
50
|
'_'
|
51
51
|
).tr('.', '_')
|
data/lib/duty_free/extensions.rb
CHANGED
@@ -6,7 +6,10 @@ require 'duty_free/suggest_template'
|
|
6
6
|
|
7
7
|
# :nodoc:
|
8
8
|
module DutyFree
|
9
|
+
# rubocop:disable Style/CommentedKeyword
|
9
10
|
module Extensions
|
11
|
+
MAX_ID = Arel.sql('MAX(id)')
|
12
|
+
|
10
13
|
def self.included(base)
|
11
14
|
base.send :extend, ClassMethods
|
12
15
|
base.send :extend, ::DutyFree::SuggestTemplate::ClassMethods
|
@@ -14,10 +17,6 @@ module DutyFree
|
|
14
17
|
|
15
18
|
# :nodoc:
|
16
19
|
module ClassMethods
|
17
|
-
MAX_ID = Arel.sql('MAX(id)')
|
18
|
-
# def self.extended(model)
|
19
|
-
# end
|
20
|
-
|
21
20
|
# Export at least column header, and optionally include all existing data as well
|
22
21
|
def df_export(is_with_data = true, import_template = nil, use_inner_joins = false)
|
23
22
|
use_inner_joins = true unless respond_to?(:left_joins)
|
@@ -53,8 +52,12 @@ module DutyFree
|
|
53
52
|
if is_with_data
|
54
53
|
order_by = []
|
55
54
|
order_by << ['_', primary_key] if primary_key
|
55
|
+
all = import_template[:all] || import_template[:all!]
|
56
56
|
# Automatically create a JOINs strategy and select list to get back all related rows
|
57
|
-
template_cols, template_joins = ::DutyFree::Extensions._recurse_def(self,
|
57
|
+
template_cols, template_joins = ::DutyFree::Extensions._recurse_def(self, all, import_template, nil, order_by)
|
58
|
+
# We do this so early here because it removes type objects from template_joins so then
|
59
|
+
# template_joins can immediately be used for inner and outer JOINs.
|
60
|
+
our_names = [[self, '_']] + ::DutyFree::Util._recurse_arel(template_joins)
|
58
61
|
relation = use_inner_joins ? joins(template_joins) : left_joins(template_joins)
|
59
62
|
|
60
63
|
# So we can properly create the SELECT list, create a mapping between our
|
@@ -71,12 +74,27 @@ module DutyFree
|
|
71
74
|
# With AR < 3.2, "froms" brings up the top node, an Arel::Nodes::InnerJoin
|
72
75
|
::DutyFree::Util._recurse_arel(core.froms)
|
73
76
|
end
|
74
|
-
our_names
|
75
|
-
|
77
|
+
# Make sure our_names lines up with the arel_alias_name by comparing the ActiveRecord type.
|
78
|
+
# AR < 5.0 behaves differently than newer versions, and AR 5.0 and 5.1 have a bug in the
|
79
|
+
# way the types get determined, so if we want perfect results then we must compensate for
|
80
|
+
# these foibles. Thank goodness that AR 5.2 and later find the proper type and make it
|
81
|
+
# available through the type_caster object, which is what we use when building the list of
|
82
|
+
# arel_alias_names.
|
83
|
+
mapping = arel_alias_names.each_with_object({}) do |arel_alias_name, s|
|
84
|
+
if our_names.first&.first == arel_alias_name.first
|
85
|
+
s[our_names.first.last] = arel_alias_name.last
|
86
|
+
our_names.shift
|
87
|
+
end
|
88
|
+
s
|
89
|
+
end
|
76
90
|
relation = (order_by.empty? ? relation : relation.order(order_by.map { |o| "#{mapping[o.first]}.#{o.last}" }))
|
77
91
|
# puts mapping.inspect
|
78
92
|
# puts relation.dup.select(template_cols.map { |x| x.to_s(mapping) }).to_sql
|
79
|
-
|
93
|
+
|
94
|
+
# Allow customisation of query before running it
|
95
|
+
relation = yield(relation, mapping) if block_given?
|
96
|
+
|
97
|
+
relation&.select(template_cols.map { |x| x.to_s(mapping) })&.each do |result|
|
80
98
|
rows << ::DutyFree::Extensions._template_columns(self, import_template).map do |col|
|
81
99
|
value = result.send(col)
|
82
100
|
case value
|
@@ -93,399 +111,57 @@ module DutyFree
|
|
93
111
|
rows
|
94
112
|
end
|
95
113
|
|
96
|
-
# With an array of incoming data, the first row having column names, perform the import
|
97
114
|
def df_import(data, import_template = nil)
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
import_template ||= if constants.include?(:IMPORT_TEMPLATE)
|
102
|
-
self::IMPORT_TEMPLATE
|
103
|
-
else
|
104
|
-
suggest_template(0, false, false)
|
105
|
-
end
|
106
|
-
# puts "Chose #{import_template}"
|
107
|
-
inserts = []
|
108
|
-
updates = []
|
109
|
-
counts = Hash.new { |h, k| h[k] = [] }
|
110
|
-
errors = []
|
111
|
-
|
112
|
-
is_first = true
|
113
|
-
uniques = nil
|
114
|
-
cols = nil
|
115
|
-
starred = []
|
116
|
-
partials = []
|
117
|
-
all = import_template[:all]
|
118
|
-
keepers = {}
|
119
|
-
valid_unique = nil
|
120
|
-
existing = {}
|
121
|
-
devise_class = ''
|
122
|
-
ret = nil
|
123
|
-
|
124
|
-
# Multi-tenancy gem Apartment can be used if there are separate schemas per tenant
|
125
|
-
reference_models = if Object.const_defined?('Apartment')
|
126
|
-
Apartment.excluded_models
|
127
|
-
else
|
128
|
-
[]
|
129
|
-
end
|
130
|
-
|
131
|
-
if Object.const_defined?('Devise')
|
132
|
-
Object.const_get('Devise') # If this fails, devise_class will remain a blank string.
|
133
|
-
devise_class = Devise.mappings.values.first.class_name
|
134
|
-
reference_models -= [devise_class]
|
135
|
-
else
|
136
|
-
devise_class = ''
|
137
|
-
end
|
138
|
-
|
139
|
-
# Did they give us a filename?
|
140
|
-
if data.is_a?(String)
|
141
|
-
# Filenames with full paths can not be longer than 4096 characters, and can not
|
142
|
-
# include newline characters
|
143
|
-
data = if data.length <= 4096 && !data.index('\n')
|
144
|
-
File.open(data)
|
145
|
-
else
|
146
|
-
# Any multi-line string is likely CSV data
|
147
|
-
# %%% Test to see if TAB characters are present on the first line, instead of commas
|
148
|
-
CSV.new(data)
|
149
|
-
end
|
150
|
-
end
|
151
|
-
# Or perhaps us a file?
|
152
|
-
if data.is_a?(File)
|
153
|
-
# Use the "roo" gem if it's available
|
154
|
-
data = if Object.const_defined?('Roo::Spreadsheet', { csv_options: { encoding: 'bom|utf-8' } })
|
155
|
-
Roo::Spreadsheet.open(data)
|
156
|
-
else
|
157
|
-
# Otherwise generic CSV parsing
|
158
|
-
require 'csv' unless Object.const_defined?('CSV')
|
159
|
-
CSV.open(data)
|
160
|
-
end
|
161
|
-
end
|
162
|
-
|
163
|
-
# Will show as just one transaction when using auditing solutions such as PaperTrail
|
164
|
-
ActiveRecord::Base.transaction do
|
165
|
-
# Check to see if they want to do anything before the whole import
|
166
|
-
# First if defined in the import_template, then if there is a method in the class,
|
167
|
-
# and finally (not yet implemented) a generic global before_import
|
168
|
-
my_before_import = import_template[:before_import]
|
169
|
-
my_before_import ||= respond_to?(:before_import) && method(:before_import)
|
170
|
-
# my_before_import ||= some generic my_before_import
|
171
|
-
if my_before_import
|
172
|
-
last_arg_idx = my_before_import.parameters.length - 1
|
173
|
-
arguments = [data, import_template][0..last_arg_idx]
|
174
|
-
data = ret if (ret = my_before_import.call(*arguments)).is_a?(Enumerable)
|
175
|
-
end
|
176
|
-
data.each_with_index do |row, row_num|
|
177
|
-
row_errors = {}
|
178
|
-
if is_first # Anticipate that first row has column names
|
179
|
-
uniques = import_template[:uniques]
|
180
|
-
|
181
|
-
# Look for UTF-8 BOM in very first cell
|
182
|
-
row[0] = row[0][3..-1] if row[0].start_with?([239, 187, 191].pack('U*'))
|
183
|
-
# How about a first character of FEFF or FFFE to support UTF-16 BOMs?
|
184
|
-
# FE FF big-endian (standard)
|
185
|
-
# FF FE little-endian
|
186
|
-
row[0] = row[0][1..-1] if [65_279, 65_534].include?(row[0][0].ord)
|
187
|
-
cols = row.map { |col| (col || '').strip }
|
188
|
-
|
189
|
-
# Unique column combinations can be called out explicitly in models using uniques: {...}, or to just
|
190
|
-
# define one column at a time simply mark with an asterisk.
|
191
|
-
# Track and clean up stars
|
192
|
-
starred = cols.select do |col|
|
193
|
-
if col[0] == '*'
|
194
|
-
col.slice!(0)
|
195
|
-
col.strip!
|
196
|
-
end
|
197
|
-
end
|
198
|
-
partials = cols.select do |col|
|
199
|
-
if col[0] == '~'
|
200
|
-
col.slice!(0)
|
201
|
-
col.strip!
|
202
|
-
end
|
203
|
-
end
|
204
|
-
cols.map! { |col| ::DutyFree::Util._clean_name(col, import_template[:as]) }
|
205
|
-
defined_uniques(uniques, cols, cols.join('|'), starred)
|
206
|
-
# Make sure that at least half of them match what we know as being good column names
|
207
|
-
template_column_objects = ::DutyFree::Extensions._recurse_def(self, import_template[:all], import_template).first
|
208
|
-
cols.each_with_index do |col, idx|
|
209
|
-
# prefixes = col_detail.pre_prefix + (col_detail.prefix.blank? ? [] : [col_detail.prefix])
|
210
|
-
keepers[idx] = template_column_objects.find { |col_obj| col_obj.titleize == col }
|
211
|
-
# puts "Could not find a match for column #{idx + 1}, #{col}" if keepers[idx].nil?
|
212
|
-
end
|
213
|
-
raise ::DutyFree::LessThanHalfAreMatchingColumnsError, I18n.t('import.altered_import_template_coumns') if keepers.length < (cols.length / 2) - 1
|
214
|
-
|
215
|
-
# Returns just the first valid unique lookup set if there are multiple
|
216
|
-
valid_unique = find_existing(uniques, cols, starred, import_template, keepers, false)
|
217
|
-
# Make a lookup from unique values to specific IDs
|
218
|
-
existing = pluck(*([:id] + valid_unique.keys)).each_with_object(existing) do |v, s|
|
219
|
-
s[v[1..-1].map(&:to_s)] = v.first
|
220
|
-
s
|
221
|
-
end
|
222
|
-
is_first = false
|
223
|
-
else # Normal row of data
|
224
|
-
is_insert = false
|
225
|
-
existing_unique = valid_unique.inject([]) do |s, v|
|
226
|
-
s << if v.last.is_a?(Array)
|
227
|
-
v.last[0].where(v.last[1] => row[v.last[2]]).limit(1).pluck(MAX_ID).first.to_s
|
228
|
-
else
|
229
|
-
row[v.last].to_s
|
230
|
-
end
|
231
|
-
end
|
232
|
-
to_be_saved = []
|
233
|
-
# Check to see if they want to preprocess anything
|
234
|
-
existing_unique = @before_process.call(valid_unique, existing_unique) if @before_process ||= import_template[:before_process]
|
235
|
-
if (criteria = existing[existing_unique])
|
236
|
-
obj = find(criteria)
|
237
|
-
else
|
238
|
-
is_insert = true
|
239
|
-
to_be_saved << [obj = new]
|
240
|
-
end
|
241
|
-
sub_obj = nil
|
242
|
-
is_has_one = false
|
243
|
-
has_ones = []
|
244
|
-
polymorphics = []
|
245
|
-
sub_objects = {}
|
246
|
-
this_path = nil
|
247
|
-
keepers.each do |key, v|
|
248
|
-
next if v.nil?
|
249
|
-
|
250
|
-
sub_obj = obj
|
251
|
-
this_path = +''
|
252
|
-
# puts "p: #{v.path}"
|
253
|
-
v.path.each_with_index do |path_part, idx|
|
254
|
-
this_path << (this_path.blank? ? path_part.to_s : ",#{path_part}")
|
255
|
-
unless (sub_next = sub_objects[this_path])
|
256
|
-
# Check if we're hitting reference data / a lookup thing
|
257
|
-
assoc = v.prefix_assocs[idx]
|
258
|
-
# belongs_to some lookup (reference) data
|
259
|
-
if assoc && reference_models.include?(assoc.class_name)
|
260
|
-
lookup_match = assoc.klass.find_by(v.name => row[key])
|
261
|
-
# Do a partial match if this column allows for it
|
262
|
-
# and we only find one matching result.
|
263
|
-
if lookup_match.nil? && partials.include?(v.titleize)
|
264
|
-
lookup_match ||= assoc.klass.where("#{v.name} LIKE '#{row[key]}%'")
|
265
|
-
lookup_match = (lookup_match.length == 1 ? lookup_match.first : nil)
|
266
|
-
end
|
267
|
-
sub_obj.send("#{path_part}=", lookup_match) unless lookup_match.nil?
|
268
|
-
# Reference data from the public level means we stop here
|
269
|
-
sub_obj = nil
|
270
|
-
break
|
271
|
-
end
|
272
|
-
# Get existing related object, or create a new one
|
273
|
-
# This first part works for belongs_to. has_many and has_one get sorted below.
|
274
|
-
if (sub_next = sub_obj.send(path_part)).nil? && assoc.belongs_to?
|
275
|
-
klass = Object.const_get(assoc&.class_name)
|
276
|
-
# Try to find a unique item if one is referenced
|
277
|
-
sub_next = nil
|
278
|
-
begin
|
279
|
-
trim_prefix = v.titleize[0..-(v.name.length + 2)]
|
280
|
-
trim_prefix << ' ' unless trim_prefix.blank?
|
281
|
-
sub_next, criteria = klass.find_existing(uniques, cols, starred, import_template, keepers, nil, row, klass, all, trim_prefix)
|
282
|
-
rescue ::DutyFree::NoUniqueColumnError
|
283
|
-
end
|
284
|
-
# puts "#{v.path} #{criteria.inspect}"
|
285
|
-
bt_name = "#{path_part}="
|
286
|
-
unless sub_next || (klass == sub_obj.class && criteria.empty?)
|
287
|
-
sub_next = klass.new(criteria || {})
|
288
|
-
to_be_saved << [sub_next, sub_obj, bt_name, sub_next]
|
289
|
-
end
|
290
|
-
# This wires it up in memory, but doesn't yet put the proper foreign key ID in
|
291
|
-
# place when the primary object is a new one (and thus not having yet been saved
|
292
|
-
# doesn't yet have an ID).
|
293
|
-
# binding.pry if !sub_next.new_record? && sub_next.name == 'Squidget Widget' # !sub_obj.changed?
|
294
|
-
# # %%% Question is -- does the above set changed? when a foreign key is not yet set
|
295
|
-
# # and only the in-memory object has changed?
|
296
|
-
is_yet_changed = sub_obj.changed?
|
297
|
-
sub_obj.send(bt_name, sub_next)
|
298
|
-
|
299
|
-
# We need this in the case of updating the primary object across a belongs_to when
|
300
|
-
# the foreign one already exists and is not otherwise changing, such as when it is
|
301
|
-
# not a new one, so wouldn't otherwise be getting saved.
|
302
|
-
to_be_saved << [sub_obj] if !sub_obj.new_record? && !is_yet_changed && sub_obj.changed?
|
303
|
-
# From a has_many or has_one?
|
304
|
-
# Rails 4.0 and later can do: sub_next.is_a?(ActiveRecord::Associations::CollectionProxy)
|
305
|
-
elsif [:has_many, :has_one].include?(assoc.macro) && !assoc.options[:through]
|
306
|
-
::DutyFree::Extensions._save_pending(to_be_saved)
|
307
|
-
# Try to find a unique item if one is referenced
|
308
|
-
# %%% There is possibility that when bringing in related classes using a nil
|
309
|
-
# in IMPORT_TEMPLATE[:all] that this will break. Need to test deeply nested things.
|
310
|
-
start = (v.pre_prefix.blank? ? 0 : v.pre_prefix.length)
|
311
|
-
trim_prefix = v.titleize[start..-(v.name.length + 2)]
|
312
|
-
trim_prefix << ' ' unless trim_prefix.blank?
|
313
|
-
# assoc.inverse_of is the belongs_to side of the has_many train we came in here on.
|
314
|
-
sub_hm, criteria = assoc.klass.find_existing(uniques, cols, starred, import_template, keepers, assoc.inverse_of, row, sub_next, all, trim_prefix)
|
315
|
-
# If still not found then create a new related object using this has_many collection
|
316
|
-
# (criteria.empty? ? nil : sub_next.new(criteria))
|
317
|
-
if sub_hm
|
318
|
-
sub_next = sub_hm
|
319
|
-
elsif assoc.macro == :has_one
|
320
|
-
# assoc.active_record.name.underscore is only there to support older Rails
|
321
|
-
# that doesn't do automatic inverse_of
|
322
|
-
ho_name = "#{assoc.inverse_of&.name || assoc.active_record.name.underscore}="
|
323
|
-
sub_next = assoc.klass.new(criteria)
|
324
|
-
to_be_saved << [sub_next, sub_next, ho_name, sub_obj]
|
325
|
-
else
|
326
|
-
# Two other methods that are possible to check for here are :conditions and
|
327
|
-
# :sanitized_conditions, which do not exist in Rails 4.0 and later.
|
328
|
-
sub_next = if assoc.respond_to?(:require_association)
|
329
|
-
# With Rails < 4.0, sub_next could come in as an Array or a broken CollectionProxy
|
330
|
-
assoc.klass.new({ fk_from(assoc) => sub_obj.send(sub_obj.class.primary_key) }.merge(criteria))
|
331
|
-
else
|
332
|
-
sub_next.proxy_association.reflection.instance_variable_set(:@foreign_key, fk_from(assoc))
|
333
|
-
sub_next.new(criteria)
|
334
|
-
end
|
335
|
-
to_be_saved << [sub_next]
|
336
|
-
end
|
337
|
-
end
|
338
|
-
# Look for possible missing polymorphic detail
|
339
|
-
# Maybe can test for this via assoc.through_reflection
|
340
|
-
if assoc.is_a?(ActiveRecord::Reflection::ThroughReflection) &&
|
341
|
-
(delegate = assoc.send(:delegate_reflection)&.active_record&.reflect_on_association(assoc.source_reflection_name)) &&
|
342
|
-
delegate.options[:polymorphic]
|
343
|
-
polymorphics << { parent: sub_next, child: sub_obj, type_col: delegate.foreign_type, id_col: delegate.foreign_key.to_s }
|
344
|
-
end
|
345
|
-
unless sub_next.nil?
|
346
|
-
# if sub_next.class.name == devise_class && # only for Devise users
|
347
|
-
# sub_next.email =~ Devise.email_regexp
|
348
|
-
# if existing.include?([sub_next.email])
|
349
|
-
# User already exists
|
350
|
-
# else
|
351
|
-
# sub_next.invite!
|
352
|
-
# end
|
353
|
-
# end
|
354
|
-
sub_objects[this_path] = sub_next if this_path.present?
|
355
|
-
end
|
356
|
-
end
|
357
|
-
sub_obj = sub_next
|
358
|
-
end
|
359
|
-
next if sub_obj.nil?
|
115
|
+
::DutyFree::Extensions.import(self, data, import_template)
|
116
|
+
end
|
360
117
|
|
361
|
-
|
118
|
+
private
|
362
119
|
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
else
|
376
|
-
row_errors[v.name] ||= []
|
377
|
-
row_errors[v.name] << "Boolean value \"#{row[key]}\" in column #{key + 1} not recognized"
|
378
|
-
end
|
379
|
-
end
|
120
|
+
def _defined_uniques(uniques, cols = [], col_list = nil, starred = [], trim_prefix = '')
|
121
|
+
col_list ||= cols.join('|')
|
122
|
+
unless (defined_uniq = (@defined_uniques ||= {})[col_list])
|
123
|
+
utilised = {} # Track columns that have been referenced thusfar
|
124
|
+
defined_uniq = uniques.each_with_object({}) do |unique, s|
|
125
|
+
if unique.is_a?(Array)
|
126
|
+
key = []
|
127
|
+
value = []
|
128
|
+
unique.each do |unique_part|
|
129
|
+
val = (unique_part_name = unique_part.to_s.titleize).start_with?(trim_prefix) &&
|
130
|
+
cols.index(upn = unique_part_name[trim_prefix.length..-1])
|
131
|
+
next unless val
|
380
132
|
|
381
|
-
|
382
|
-
|
383
|
-
# If this one is transitioning from having not changed to now having been changed,
|
384
|
-
# and is not a new one anyway that would already be lined up to get saved, then we
|
385
|
-
# mark it to now be saved.
|
386
|
-
to_be_saved << [sub_obj] if !sub_obj.new_record? && !is_yet_changed && sub_obj.changed?
|
387
|
-
# else
|
388
|
-
# puts " #{sub_obj.class.name} doesn't respond to #{sym}"
|
133
|
+
key << upn
|
134
|
+
value << val
|
389
135
|
end
|
390
|
-
|
391
|
-
|
392
|
-
|
393
|
-
# Reinstate any missing polymorphic _type and _id values
|
394
|
-
polymorphics.each do |poly|
|
395
|
-
if !poly[:parent].new_record? || poly[:parent].save
|
396
|
-
poly[:child].send("#{poly[:type_col]}=".to_sym, poly[:parent].class.name)
|
397
|
-
poly[:child].send("#{poly[:id_col]}=".to_sym, poly[:parent].id)
|
398
|
-
end
|
136
|
+
unless key.empty?
|
137
|
+
s[key] = value
|
138
|
+
utilised[key] = nil
|
399
139
|
end
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
true
|
407
|
-
end
|
408
|
-
|
409
|
-
if obj.valid?
|
410
|
-
obj.save if is_do_save
|
411
|
-
# Snag back any changes to the unique columns. (For instance, Devise downcases email addresses.)
|
412
|
-
existing_unique = valid_unique.keys.inject([]) { |s, v| s << obj.send(v).to_s }
|
413
|
-
# Update the duplicate counts and inserted / updated results
|
414
|
-
counts[existing_unique] << row_num
|
415
|
-
(is_insert ? inserts : updates) << { row_num => existing_unique } if is_do_save
|
416
|
-
# Track this new object so we can properly sense any duplicates later
|
417
|
-
existing[existing_unique] = obj.id
|
418
|
-
else
|
419
|
-
row_errors.merge! obj.errors.messages
|
140
|
+
else
|
141
|
+
val = (unique_name = unique.to_s.titleize).start_with?(trim_prefix) &&
|
142
|
+
cols.index(un = unique_name[trim_prefix.length..-1])
|
143
|
+
if val
|
144
|
+
s[[un]] = [val]
|
145
|
+
utilised[[un]] = nil
|
420
146
|
end
|
421
|
-
errors << { row_num => row_errors } unless row_errors.empty?
|
422
147
|
end
|
423
|
-
end
|
424
|
-
duplicates = counts.each_with_object([]) do |v, s|
|
425
|
-
s += v.last[1..-1].map { |line_num| { line_num => v.first } } if v.last.count > 1
|
426
148
|
s
|
427
149
|
end
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
# First if defined in the import_template, then if there is a method in the class,
|
432
|
-
# and finally (not yet implemented) a generic global after_import
|
433
|
-
my_after_import = import_template[:after_import]
|
434
|
-
my_after_import ||= respond_to?(:after_import) && method(:after_import)
|
435
|
-
# my_after_import ||= some generic my_after_import
|
436
|
-
if my_after_import
|
437
|
-
last_arg_idx = my_after_import.parameters.length - 1
|
438
|
-
arguments = [ret][0..last_arg_idx]
|
439
|
-
ret = ret2 if (ret2 = my_after_import.call(*arguments)).is_a?(Hash)
|
440
|
-
end
|
441
|
-
end
|
442
|
-
ret
|
443
|
-
end
|
444
|
-
|
445
|
-
def fk_from(assoc)
|
446
|
-
# Try first to trust whatever they've marked as being the foreign_key, and then look
|
447
|
-
# at the inverse's foreign key setting if available. In all cases don't accept
|
448
|
-
# anything that's not backed with a real column in the table.
|
449
|
-
col_names = assoc.klass.column_names
|
450
|
-
if (fk_name = assoc.options[:foreign_key]) && col_names.include?(fk_name.to_s)
|
451
|
-
return fk_name
|
452
|
-
end
|
453
|
-
if (fk_name = assoc.inverse_of&.options&.fetch(:foreign_key) { nil }) &&
|
454
|
-
col_names.include?(fk_name.to_s)
|
455
|
-
return fk_name
|
456
|
-
end
|
457
|
-
if assoc.respond_to?(:foreign_key) && (fk_name = assoc.foreign_key) &&
|
458
|
-
col_names.include?(fk_name.to_s)
|
459
|
-
return fk_name
|
460
|
-
end
|
461
|
-
if (fk_name = assoc.inverse_of&.foreign_key) &&
|
462
|
-
col_names.include?(fk_name.to_s)
|
463
|
-
return fk_name
|
464
|
-
end
|
465
|
-
if (fk_name = assoc.inverse_of&.association_foreign_key) &&
|
466
|
-
col_names.include?(fk_name.to_s)
|
467
|
-
return fk_name
|
468
|
-
end
|
469
|
-
# Don't let this fool you -- we really are in search of the foreign key name here,
|
470
|
-
# and Rails 3.0 and older used some fairly interesting conventions, calling it instead
|
471
|
-
# the "primary_key_name"!
|
472
|
-
if assoc.respond_to?(:primary_key_name)
|
473
|
-
if (fk_name = assoc.primary_key_name) && col_names.include?(fk_name.to_s)
|
474
|
-
return fk_name
|
475
|
-
end
|
476
|
-
if (fk_name = assoc.inverse_of.primary_key_name) && col_names.include?(fk_name.to_s)
|
477
|
-
return fk_name
|
150
|
+
if defined_uniq.empty?
|
151
|
+
(starred - utilised.keys).each { |star| defined_uniq[[star]] = [cols.index(star)] }
|
152
|
+
# %%% puts "Tried to establish #{defined_uniq.inspect}"
|
478
153
|
end
|
154
|
+
@defined_uniques[col_list] = defined_uniq
|
479
155
|
end
|
480
|
-
|
481
|
-
puts "* Wow, had no luck at all finding a foreign key for #{assoc.inspect}"
|
156
|
+
defined_uniq
|
482
157
|
end
|
483
158
|
|
484
159
|
# For use with importing, based on the provided column list calculate all valid combinations
|
485
160
|
# of unique columns. If there is no valid combination, throws an error.
|
486
|
-
# Returns an object found by this means.
|
487
|
-
def
|
488
|
-
|
161
|
+
# Returns an object found by this means, as well as the criteria that was used to find it.
|
162
|
+
def _find_existing(uniques, cols, starred, import_template, keepers, train_we_came_in_here_on,
|
163
|
+
row = nil, klass_or_collection = nil, template_all = nil, trim_prefix = '',
|
164
|
+
assoc = nil, base_obj = nil)
|
489
165
|
unless trim_prefix.blank?
|
490
166
|
cols = cols.map { |c| c.start_with?(trim_prefix) ? c[trim_prefix.length..-1] : nil }
|
491
167
|
starred = starred.each_with_object([]) do |v, s|
|
@@ -536,7 +212,7 @@ module DutyFree
|
|
536
212
|
# folks based on having their email as a unique identifier, and other folks by having a
|
537
213
|
# combination of their name and street address as unique, and use both of those possible
|
538
214
|
# unique variations to update phone numbers, and do that all as a part of one import.)
|
539
|
-
all_vus =
|
215
|
+
all_vus = _defined_uniques(uniques, cols, col_list, starred, trim_prefix)
|
540
216
|
|
541
217
|
# %%% Ultimately may consider making this recursive
|
542
218
|
reflect_on_all_associations.each do |sn_bt|
|
@@ -662,13 +338,56 @@ module DutyFree
|
|
662
338
|
end
|
663
339
|
|
664
340
|
return uniq_lookups.merge(criteria) if only_valid_uniques
|
665
|
-
|
666
341
|
# If there's nothing to match upon then we're out
|
667
342
|
return [nil, {}] if new_criteria_all_nil
|
668
343
|
|
669
344
|
# With this criteria, find any matching has_many row we can so we can update it.
|
670
345
|
# First try directly looking it up through ActiveRecord.
|
671
|
-
|
346
|
+
klass_or_collection ||= self # Comes in as nil for has_one with no object yet attached
|
347
|
+
# HABTM proxy
|
348
|
+
# binding.pry if klass_or_collection.is_a?(ActiveRecord::Associations::CollectionProxy)
|
349
|
+
# if klass_or_collection.respond_to?(:proxy_association) && klass_or_collection.proxy_association.options.include?(:through)
|
350
|
+
# klass_or_collection.proxy_association.association_scope.to_a
|
351
|
+
|
352
|
+
# if assoc.respond_to?(:require_association) && klass_or_collection.is_a?(Array)
|
353
|
+
# # else
|
354
|
+
# # klass_or_collection = assoc.klass
|
355
|
+
# end
|
356
|
+
# end
|
357
|
+
found_object = case klass_or_collection
|
358
|
+
when ActiveRecord::Base # object from a has_one?
|
359
|
+
existing_object = klass_or_collection
|
360
|
+
klass_or_collection = klass_or_collection.class
|
361
|
+
other_object = klass_or_collection.find_by(criteria)
|
362
|
+
pk = klass_or_collection.primary_key
|
363
|
+
existing_object.send(pk) == other_object&.send(pk) ? existing_object : other_object
|
364
|
+
when Array # has_* in AR < 4.0
|
365
|
+
# Old AR doesn't have a CollectionProxy that can do any of this on its own.
|
366
|
+
base_id = base_obj.send(base_obj.class.primary_key)
|
367
|
+
if assoc.macro == :has_and_belongs_to_many || (assoc.macro == :has_many && assoc.options[:through])
|
368
|
+
# Find all association foreign keys, then find or create the foreign object
|
369
|
+
# based on criteria, and finally put an entry with both foreign keys into
|
370
|
+
# the associative table unless it already exists.
|
371
|
+
ajt = assoc.through_reflection&.table_name || assoc.join_table
|
372
|
+
fk = assoc.foreign_key
|
373
|
+
afk = assoc.association_foreign_key
|
374
|
+
existing_ids = ActiveRecord::Base.connection.execute(
|
375
|
+
"SELECT #{afk} FROM #{ajt} WHERE #{fk} = #{base_id}"
|
376
|
+
).map { |r| r[afk] }
|
377
|
+
new_or_existing = assoc.klass.find_or_create_by(criteria)
|
378
|
+
new_or_existing_id = new_or_existing.send(new_or_existing.class.primary_key)
|
379
|
+
unless existing_ids.include?(new_or_existing_id)
|
380
|
+
ActiveRecord::Base.connection.execute(
|
381
|
+
"INSERT INTO #{ajt} (#{fk}, #{afk}) VALUES (#{base_id}, #{new_or_existing_id})"
|
382
|
+
)
|
383
|
+
end
|
384
|
+
new_or_existing
|
385
|
+
else # Must be a has_many
|
386
|
+
assoc.klass.find_or_create_by(criteria.merge({ assoc.foreign_key => base_id }))
|
387
|
+
end
|
388
|
+
else
|
389
|
+
klass_or_collection.find_by(criteria)
|
390
|
+
end
|
672
391
|
# If not successful, such as when fields are exposed via helper methods instead of being
|
673
392
|
# real columns in the database tables, try this more intensive approach. This is useful
|
674
393
|
# if you had full name kind of data coming in on a spreadsheeet, but in the destination
|
@@ -692,58 +411,445 @@ module DutyFree
|
|
692
411
|
# that match up to a primary key so that if needed a new related object can be built,
|
693
412
|
# complete with all its association detail.
|
694
413
|
[found_object, criteria.merge(bt_criteria)]
|
695
|
-
end
|
414
|
+
end # _find_existing
|
415
|
+
end # module ClassMethods
|
696
416
|
|
697
|
-
|
417
|
+
# With an array of incoming data, the first row having column names, perform the import
|
418
|
+
def self.import(obj_klass, data, import_template = nil)
|
419
|
+
instance_variable_set(:@defined_uniques, nil)
|
420
|
+
instance_variable_set(:@valid_uniques, nil)
|
421
|
+
|
422
|
+
import_template ||= if obj_klass.constants.include?(:IMPORT_TEMPLATE)
|
423
|
+
obj_klass::IMPORT_TEMPLATE
|
424
|
+
else
|
425
|
+
obj_klass.suggest_template(0, false, false)
|
426
|
+
end
|
427
|
+
# puts "Chose #{import_template}"
|
428
|
+
inserts = []
|
429
|
+
updates = []
|
430
|
+
counts = Hash.new { |h, k| h[k] = [] }
|
431
|
+
errors = []
|
432
|
+
|
433
|
+
is_first = true
|
434
|
+
uniques = nil
|
435
|
+
cols = nil
|
436
|
+
starred = []
|
437
|
+
partials = []
|
438
|
+
# See if we can find the model if given only a string
|
439
|
+
if obj_klass.is_a?(String)
|
440
|
+
obj_klass_camelized = obj_klass.camelize.singularize
|
441
|
+
obj_klass = Object.const_get(obj_klass_camelized) if Object.const_defined?(obj_klass_camelized)
|
442
|
+
end
|
443
|
+
table_name = obj_klass.table_name unless obj_klass.is_a?(String)
|
444
|
+
is_build_table = if import_template.include?(:all!)
|
445
|
+
# Search for presence of this table
|
446
|
+
table_name = obj_klass.underscore.pluralize if obj_klass.is_a?(String)
|
447
|
+
!ActiveRecord::ConnectionAdapters::SchemaStatements.table_exists?(table_name)
|
448
|
+
else
|
449
|
+
false
|
450
|
+
end
|
451
|
+
# If we do need to build the table then we require an :all!, otherwise in the case that the table
|
452
|
+
# does not need to be built or was already built then try :all, and fall back on :all!.
|
453
|
+
all = import_template[is_build_table ? :all! : :all] || import_template[:all!]
|
454
|
+
keepers = {}
|
455
|
+
valid_unique = nil
|
456
|
+
existing = {}
|
457
|
+
devise_class = ''
|
458
|
+
ret = nil
|
459
|
+
|
460
|
+
# Multi-tenancy gem Apartment can be used if there are separate schemas per tenant
|
461
|
+
reference_models = if Object.const_defined?('Apartment')
|
462
|
+
Apartment.excluded_models
|
463
|
+
else
|
464
|
+
[]
|
465
|
+
end
|
466
|
+
|
467
|
+
if Object.const_defined?('Devise')
|
468
|
+
Object.const_get('Devise') # If this fails, devise_class will remain a blank string.
|
469
|
+
devise_class = Devise.mappings.values.first.class_name
|
470
|
+
reference_models -= [devise_class]
|
471
|
+
else
|
472
|
+
devise_class = ''
|
473
|
+
end
|
698
474
|
|
699
|
-
|
700
|
-
|
701
|
-
|
702
|
-
|
703
|
-
|
704
|
-
|
705
|
-
|
706
|
-
|
707
|
-
|
708
|
-
|
709
|
-
|
710
|
-
|
475
|
+
# Did they give us a filename?
|
476
|
+
if data.is_a?(String)
|
477
|
+
# Filenames with full paths can not be longer than 4096 characters, and can not
|
478
|
+
# include newline characters
|
479
|
+
data = if data.length <= 4096 && !data.index('\n')
|
480
|
+
File.open(data)
|
481
|
+
else
|
482
|
+
# Any multi-line string is likely CSV data
|
483
|
+
# %%% Test to see if TAB characters are present on the first line, instead of commas
|
484
|
+
CSV.new(data)
|
485
|
+
end
|
486
|
+
end
|
487
|
+
# Or perhaps us a file?
|
488
|
+
if data.is_a?(File)
|
489
|
+
# Use the "roo" gem if it's available
|
490
|
+
# When we're ready to try parsing this thing on our own, shared strings and all, then use
|
491
|
+
# the rubyzip gem also along with it:
|
492
|
+
# https://github.com/rubyzip/rubyzip
|
493
|
+
# require 'zip'
|
494
|
+
data = if Object.const_defined?('Roo::Spreadsheet', { csv_options: { encoding: 'bom|utf-8' } })
|
495
|
+
Roo::Spreadsheet.open(data)
|
496
|
+
else
|
497
|
+
# Otherwise generic CSV parsing
|
498
|
+
require 'csv' unless Object.const_defined?('CSV')
|
499
|
+
CSV.open(data)
|
500
|
+
end
|
501
|
+
end
|
711
502
|
|
712
|
-
|
713
|
-
|
503
|
+
# Will show as just one transaction when using auditing solutions such as PaperTrail
|
504
|
+
ActiveRecord::Base.transaction do
|
505
|
+
# Check to see if they want to do anything before the whole import
|
506
|
+
# First if defined in the import_template, then if there is a method in the class,
|
507
|
+
# and finally (not yet implemented) a generic global before_import
|
508
|
+
my_before_import = import_template[:before_import]
|
509
|
+
my_before_import ||= respond_to?(:before_import) && method(:before_import)
|
510
|
+
# my_before_import ||= some generic my_before_import
|
511
|
+
if my_before_import
|
512
|
+
last_arg_idx = my_before_import.parameters.length - 1
|
513
|
+
arguments = [data, import_template][0..last_arg_idx]
|
514
|
+
data = ret if (ret = my_before_import.call(*arguments)).is_a?(Enumerable)
|
515
|
+
end
|
516
|
+
build_tables = nil
|
517
|
+
data.each_with_index do |row, row_num|
|
518
|
+
row_errors = {}
|
519
|
+
if is_first # Anticipate that first row has column names
|
520
|
+
uniques = import_template[:uniques]
|
521
|
+
|
522
|
+
# Look for UTF-8 BOM in very first cell
|
523
|
+
row[0] = row[0][3..-1] if row[0].start_with?([239, 187, 191].pack('U*'))
|
524
|
+
# How about a first character of FEFF or FFFE to support UTF-16 BOMs?
|
525
|
+
# FE FF big-endian (standard)
|
526
|
+
# FF FE little-endian
|
527
|
+
row[0] = row[0][1..-1] if [65_279, 65_534].include?(row[0][0].ord)
|
528
|
+
cols = row.map { |col| (col || '').strip }
|
529
|
+
|
530
|
+
# Unique column combinations can be called out explicitly in models using uniques: {...}, or to just
|
531
|
+
# define one column at a time simply mark with an asterisk.
|
532
|
+
# Track and clean up stars
|
533
|
+
starred = cols.select do |col|
|
534
|
+
if col[0] == '*'
|
535
|
+
col.slice!(0)
|
536
|
+
col.strip!
|
714
537
|
end
|
715
|
-
|
716
|
-
|
717
|
-
|
538
|
+
end
|
539
|
+
partials = cols.select do |col|
|
540
|
+
if col[0] == '~'
|
541
|
+
col.slice!(0)
|
542
|
+
col.strip!
|
718
543
|
end
|
544
|
+
end
|
545
|
+
cols.map! { |col| ::DutyFree::Util._clean_name(col, import_template[:as]) }
|
546
|
+
obj_klass.send(:_defined_uniques, uniques, cols, cols.join('|'), starred)
|
547
|
+
# Main object asking for table to be built?
|
548
|
+
build_tables = {}
|
549
|
+
build_tables[path_name] = [namespaces, is_need_model, is_need_table] if is_build_table
|
550
|
+
# Make sure that at least half of them match what we know as being good column names
|
551
|
+
template_column_objects = ::DutyFree::Extensions._recurse_def(obj_klass, import_template[:all], import_template, build_tables).first
|
552
|
+
cols.each_with_index do |col, idx|
|
553
|
+
# prefixes = col_detail.pre_prefix + (col_detail.prefix.blank? ? [] : [col_detail.prefix])
|
554
|
+
# %%% Would be great here if when one comes back nil, try to find the closest match
|
555
|
+
# and indicate to the user a "did you mean?" about it.
|
556
|
+
keepers[idx] = template_column_objects.find { |col_obj| col_obj.titleize == col }
|
557
|
+
# puts "Could not find a match for column #{idx + 1}, #{col}" if keepers[idx].nil?
|
558
|
+
end
|
559
|
+
raise ::DutyFree::LessThanHalfAreMatchingColumnsError, I18n.t('import.altered_import_template_coumns') if keepers.length < (cols.length / 2) - 1
|
560
|
+
|
561
|
+
# Returns just the first valid unique lookup set if there are multiple
|
562
|
+
valid_unique = obj_klass.send(:_find_existing, uniques, cols, starred, import_template, keepers, false)
|
563
|
+
# Make a lookup from unique values to specific IDs
|
564
|
+
existing = obj_klass.pluck(*([:id] + valid_unique.keys)).each_with_object(existing) do |v, s|
|
565
|
+
s[v[1..-1].map(&:to_s)] = v.first
|
566
|
+
s
|
567
|
+
end
|
568
|
+
is_first = false
|
569
|
+
else # Normal row of data
|
570
|
+
is_insert = false
|
571
|
+
existing_unique = valid_unique.inject([]) do |s, v|
|
572
|
+
s << if v.last.is_a?(Array)
|
573
|
+
v.last[0].where(v.last[1] => row[v.last[2]]).limit(1).pluck(MAX_ID).first.to_s
|
574
|
+
else
|
575
|
+
row[v.last].to_s
|
576
|
+
end
|
577
|
+
end
|
578
|
+
to_be_saved = []
|
579
|
+
# Check to see if they want to preprocess anything
|
580
|
+
existing_unique = @before_process.call(valid_unique, existing_unique) if @before_process ||= import_template[:before_process]
|
581
|
+
if (criteria = existing[existing_unique])
|
582
|
+
obj = obj_klass.find(criteria)
|
719
583
|
else
|
720
|
-
|
721
|
-
|
722
|
-
|
723
|
-
|
724
|
-
|
584
|
+
is_insert = true
|
585
|
+
# unless build_tables.empty? # include?()
|
586
|
+
# binding.pry
|
587
|
+
# x = 5
|
588
|
+
# end
|
589
|
+
to_be_saved << [obj = obj_klass.new]
|
590
|
+
end
|
591
|
+
sub_obj = nil
|
592
|
+
polymorphics = []
|
593
|
+
sub_objects = {}
|
594
|
+
this_path = nil
|
595
|
+
keepers.each do |key, v|
|
596
|
+
next if v.nil?
|
597
|
+
|
598
|
+
sub_obj = obj
|
599
|
+
this_path = +''
|
600
|
+
# puts "p: #{v.path}"
|
601
|
+
v.path.each_with_index do |path_part, idx|
|
602
|
+
this_path << (this_path.blank? ? path_part.to_s : ",#{path_part}")
|
603
|
+
unless (sub_next = sub_objects[this_path])
|
604
|
+
# Check if we're hitting reference data / a lookup thing
|
605
|
+
assoc = v.prefix_assocs[idx]
|
606
|
+
# belongs_to some lookup (reference) data
|
607
|
+
if assoc && reference_models.include?(assoc.class_name)
|
608
|
+
lookup_match = assoc.klass.find_by(v.name => row[key])
|
609
|
+
# Do a partial match if this column allows for it
|
610
|
+
# and we only find one matching result.
|
611
|
+
if lookup_match.nil? && partials.include?(v.titleize)
|
612
|
+
lookup_match ||= assoc.klass.where("#{v.name} LIKE '#{row[key]}%'")
|
613
|
+
lookup_match = (lookup_match.length == 1 ? lookup_match.first : nil)
|
614
|
+
end
|
615
|
+
sub_obj.send("#{path_part}=", lookup_match) unless lookup_match.nil?
|
616
|
+
# Reference data from the public level means we stop here
|
617
|
+
sub_obj = nil
|
618
|
+
break
|
619
|
+
end
|
620
|
+
# Get existing related object, or create a new one
|
621
|
+
# This first part works for belongs_to. has_many and has_one get sorted below.
|
622
|
+
# start = (v.pre_prefix.blank? ? 0 : v.pre_prefix.length)
|
623
|
+
start = 0
|
624
|
+
trim_prefix = v.titleize[start..-(v.name.length + 2)]
|
625
|
+
trim_prefix << ' ' unless trim_prefix.blank?
|
626
|
+
if (sub_next = sub_obj.send(path_part)).nil? && assoc.belongs_to?
|
627
|
+
klass = Object.const_get(assoc&.class_name)
|
628
|
+
# Try to find a unique item if one is referenced
|
629
|
+
sub_next = nil
|
630
|
+
begin
|
631
|
+
sub_next, criteria = klass.send(:_find_existing, uniques, cols, starred, import_template, keepers, nil, row, klass, all, trim_prefix, assoc)
|
632
|
+
rescue ::DutyFree::NoUniqueColumnError
|
633
|
+
end
|
634
|
+
# puts "#{v.path} #{criteria.inspect}"
|
635
|
+
bt_name = "#{path_part}="
|
636
|
+
unless sub_next || (klass == sub_obj.class && criteria.empty?)
|
637
|
+
sub_next = klass.new(criteria || {})
|
638
|
+
to_be_saved << [sub_next, sub_obj, bt_name, sub_next]
|
639
|
+
end
|
640
|
+
# This wires it up in memory, but doesn't yet put the proper foreign key ID in
|
641
|
+
# place when the primary object is a new one (and thus not having yet been saved
|
642
|
+
# doesn't yet have an ID).
|
643
|
+
# binding.pry if !sub_next.new_record? && sub_next.name == 'Squidget Widget' # !sub_obj.changed?
|
644
|
+
# # %%% Question is -- does the above set changed? when a foreign key is not yet set
|
645
|
+
# # and only the in-memory object has changed?
|
646
|
+
is_yet_changed = sub_obj.changed?
|
647
|
+
sub_obj.send(bt_name, sub_next)
|
648
|
+
|
649
|
+
# We need this in the case of updating the primary object across a belongs_to when
|
650
|
+
# the foreign one already exists and is not otherwise changing, such as when it is
|
651
|
+
# not a new one, so wouldn't otherwise be getting saved.
|
652
|
+
to_be_saved << [sub_obj] if !sub_obj.new_record? && !is_yet_changed && sub_obj.changed?
|
653
|
+
# From a has_many or has_one?
|
654
|
+
# Rails 4.0 and later can do: sub_next.is_a?(ActiveRecord::Associations::CollectionProxy)
|
655
|
+
elsif [:has_many, :has_one, :has_and_belongs_to_many].include?(assoc.macro) # && !assoc.options[:through]
|
656
|
+
::DutyFree::Extensions._save_pending(to_be_saved)
|
657
|
+
# Try to find a unique item if one is referenced
|
658
|
+
# %%% There is possibility that when bringing in related classes using a nil
|
659
|
+
# in IMPORT_TEMPLATE[:all] that this will break. Need to test deeply nested things.
|
660
|
+
|
661
|
+
# assoc.inverse_of is the belongs_to side of the has_many train we came in here on.
|
662
|
+
sub_hm, criteria = assoc.klass.send(:_find_existing, uniques, cols, starred, import_template, keepers, assoc.inverse_of,
|
663
|
+
row, sub_next, all, trim_prefix, assoc,
|
664
|
+
# Just in case we're running Rails < 4.0 and this is a haas_*
|
665
|
+
sub_obj)
|
666
|
+
# If still not found then create a new related object using this has_many collection
|
667
|
+
# (criteria.empty? ? nil : sub_next.new(criteria))
|
668
|
+
if sub_hm
|
669
|
+
sub_next = sub_hm
|
670
|
+
elsif assoc.macro == :has_one
|
671
|
+
# assoc.active_record.name.underscore is only there to support older Rails
|
672
|
+
# that doesn't do automatic inverse_of
|
673
|
+
ho_name = "#{assoc.inverse_of&.name || assoc.active_record.name.underscore}="
|
674
|
+
sub_next = assoc.klass.new(criteria)
|
675
|
+
to_be_saved << [sub_next, sub_next, ho_name, sub_obj]
|
676
|
+
elsif assoc.macro == :has_and_belongs_to_many ||
|
677
|
+
(assoc.macro == :has_many && assoc.options[:through])
|
678
|
+
# sub_next = sub_next.new(criteria)
|
679
|
+
# Search for one to wire up if it might already exist, otherwise create one
|
680
|
+
sub_next = assoc.klass.find_by(criteria) || assoc.klass.new(criteria)
|
681
|
+
to_be_saved << [sub_next, :has_and_belongs_to_many, assoc.name, sub_obj]
|
682
|
+
else
|
683
|
+
# Two other methods that are possible to check for here are :conditions and
|
684
|
+
# :sanitized_conditions, which do not exist in Rails 4.0 and later.
|
685
|
+
sub_next = if assoc.respond_to?(:require_association)
|
686
|
+
# With Rails < 4.0, sub_next could come in as an Array or a broken CollectionProxy
|
687
|
+
assoc.klass.new({ ::DutyFree::Extensions._fk_from(assoc) => sub_obj.send(sub_obj.class.primary_key) }.merge(criteria))
|
688
|
+
else
|
689
|
+
sub_next.proxy_association.reflection.instance_variable_set(:@foreign_key, ::DutyFree::Extensions._fk_from(assoc))
|
690
|
+
sub_next.new(criteria)
|
691
|
+
end
|
692
|
+
to_be_saved << [sub_next]
|
693
|
+
end
|
694
|
+
# else
|
695
|
+
# belongs_to for a found object, or a HMT
|
696
|
+
end
|
697
|
+
# # Incompatible with Rails < 4.2
|
698
|
+
# # Look for possible missing polymorphic detail
|
699
|
+
# # Maybe can test for this via assoc.through_reflection
|
700
|
+
# if assoc.is_a?(ActiveRecord::Reflection::ThroughReflection) &&
|
701
|
+
# (delegate = assoc.send(:delegate_reflection)&.active_record&.reflect_on_association(assoc.source_reflection_name)) &&
|
702
|
+
# delegate.options[:polymorphic]
|
703
|
+
# polymorphics << { parent: sub_next, child: sub_obj, type_col: delegate.foreign_type, id_col: delegate.foreign_key.to_s }
|
704
|
+
# end
|
705
|
+
|
706
|
+
# rubocop:disable Style/SoleNestedConditional
|
707
|
+
unless sub_next.nil?
|
708
|
+
# if sub_next.class.name == devise_class && # only for Devise users
|
709
|
+
# sub_next.email =~ Devise.email_regexp
|
710
|
+
# if existing.include?([sub_next.email])
|
711
|
+
# User already exists
|
712
|
+
# else
|
713
|
+
# sub_next.invite!
|
714
|
+
# end
|
715
|
+
# end
|
716
|
+
sub_objects[this_path] = sub_next if this_path.present?
|
717
|
+
end
|
718
|
+
# rubocop:enable Style/SoleNestedConditional
|
719
|
+
end
|
720
|
+
sub_obj = sub_next
|
721
|
+
end
|
722
|
+
next if sub_obj.nil?
|
723
|
+
|
724
|
+
next unless sub_obj.respond_to?(sym = "#{v.name}=".to_sym)
|
725
|
+
|
726
|
+
col_type = sub_obj.class.columns_hash[v.name.to_s]&.type
|
727
|
+
if col_type.nil? && (virtual_columns = import_template[:virtual_columns]) &&
|
728
|
+
(virtual_columns = virtual_columns[this_path] || virtual_columns)
|
729
|
+
col_type = virtual_columns[v.name]
|
725
730
|
end
|
731
|
+
if col_type == :boolean
|
732
|
+
if row[key].nil?
|
733
|
+
# Do nothing when it's nil
|
734
|
+
elsif %w[true t yes y].include?(row[key]&.strip&.downcase) # Used to cover 'on'
|
735
|
+
row[key] = true
|
736
|
+
elsif %w[false f no n].include?(row[key]&.strip&.downcase) # Used to cover 'off'
|
737
|
+
row[key] = false
|
738
|
+
else
|
739
|
+
row_errors[v.name] ||= []
|
740
|
+
row_errors[v.name] << "Boolean value \"#{row[key]}\" in column #{key + 1} not recognized"
|
741
|
+
end
|
742
|
+
end
|
743
|
+
|
744
|
+
is_yet_changed = sub_obj.changed?
|
745
|
+
sub_obj.send(sym, row[key])
|
746
|
+
# If this one is transitioning from having not changed to now having been changed,
|
747
|
+
# and is not a new one anyway that would already be lined up to get saved, then we
|
748
|
+
# mark it to now be saved.
|
749
|
+
to_be_saved << [sub_obj] if !sub_obj.new_record? && !is_yet_changed && sub_obj.changed?
|
750
|
+
# else
|
751
|
+
# puts " #{sub_obj.class.name} doesn't respond to #{sym}"
|
726
752
|
end
|
727
|
-
s
|
728
|
-
|
729
|
-
|
730
|
-
|
731
|
-
|
753
|
+
# Try to save final sub-object(s) if any exist
|
754
|
+
::DutyFree::Extensions._save_pending(to_be_saved)
|
755
|
+
|
756
|
+
# Reinstate any missing polymorphic _type and _id values
|
757
|
+
polymorphics.each do |poly|
|
758
|
+
if !poly[:parent].new_record? || poly[:parent].save
|
759
|
+
poly[:child].send("#{poly[:type_col]}=".to_sym, poly[:parent].class.name)
|
760
|
+
poly[:child].send("#{poly[:id_col]}=".to_sym, poly[:parent].id)
|
761
|
+
end
|
762
|
+
end
|
763
|
+
|
764
|
+
# Give a window of opportunity to tweak user objects controlled by Devise
|
765
|
+
obj_class = obj.class
|
766
|
+
is_do_save = if obj_class.respond_to?(:before_devise_save) && obj_class.name == devise_class
|
767
|
+
obj_class.before_devise_save(obj, existing)
|
768
|
+
else
|
769
|
+
true
|
770
|
+
end
|
771
|
+
|
772
|
+
if obj.valid?
|
773
|
+
obj.save if is_do_save
|
774
|
+
# Snag back any changes to the unique columns. (For instance, Devise downcases email addresses.)
|
775
|
+
existing_unique = valid_unique.keys.inject([]) { |s, v| s << obj.send(v).to_s }
|
776
|
+
# Update the duplicate counts and inserted / updated results
|
777
|
+
counts[existing_unique] << row_num
|
778
|
+
(is_insert ? inserts : updates) << { row_num => existing_unique } if is_do_save
|
779
|
+
# Track this new object so we can properly sense any duplicates later
|
780
|
+
existing[existing_unique] = obj.id
|
781
|
+
else
|
782
|
+
row_errors.merge! obj.errors.messages
|
783
|
+
end
|
784
|
+
errors << { row_num => row_errors } unless row_errors.empty?
|
732
785
|
end
|
733
|
-
@defined_uniques[col_list] = defined_uniq
|
734
786
|
end
|
735
|
-
|
787
|
+
duplicates = counts.each_with_object([]) do |v, s|
|
788
|
+
s += v.last[1..-1].map { |line_num| { line_num => v.first } } if v.last.count > 1
|
789
|
+
s
|
790
|
+
end
|
791
|
+
ret = { inserted: inserts, updated: updates, duplicates: duplicates, errors: errors }
|
792
|
+
|
793
|
+
# Check to see if they want to do anything after the import
|
794
|
+
# First if defined in the import_template, then if there is a method in the class,
|
795
|
+
# and finally (not yet implemented) a generic global after_import
|
796
|
+
my_after_import = import_template[:after_import]
|
797
|
+
my_after_import ||= respond_to?(:after_import) && method(:after_import)
|
798
|
+
# my_after_import ||= some generic my_after_import
|
799
|
+
if my_after_import
|
800
|
+
last_arg_idx = my_after_import.parameters.length - 1
|
801
|
+
arguments = [ret][0..last_arg_idx]
|
802
|
+
# rubocop:disable Lint/UselessAssignment
|
803
|
+
ret = ret2 if (ret2 = my_after_import.call(*arguments)).is_a?(Hash)
|
804
|
+
# rubocop:enable Lint/UselessAssignment
|
805
|
+
end
|
736
806
|
end
|
737
|
-
|
807
|
+
ret
|
808
|
+
end
|
809
|
+
|
810
|
+
def self._fk_from(assoc)
|
811
|
+
# Try first to trust whatever they've marked as being the foreign_key, and then look
|
812
|
+
# at the inverse's foreign key setting if available. In all cases don't accept
|
813
|
+
# anything that's not backed with a real column in the table.
|
814
|
+
col_names = assoc.klass.column_names
|
815
|
+
if (
|
816
|
+
(fk_name = assoc.options[:foreign_key]) ||
|
817
|
+
(fk_name = assoc.inverse_of&.options&.fetch(:foreign_key) { nil }) ||
|
818
|
+
(assoc.respond_to?(:foreign_key) && (fk_name = assoc.foreign_key)) ||
|
819
|
+
(fk_name = assoc.inverse_of&.foreign_key) ||
|
820
|
+
(fk_name = assoc.inverse_of&.association_foreign_key)
|
821
|
+
) && col_names.include?(fk_name.to_s)
|
822
|
+
return fk_name
|
823
|
+
end
|
824
|
+
|
825
|
+
# Don't let this fool you -- we really are in search of the foreign key name here,
|
826
|
+
# and Rails 3.0 and older used some fairly interesting conventions, calling it instead
|
827
|
+
# the "primary_key_name"!
|
828
|
+
if assoc.respond_to?(:primary_key_name)
|
829
|
+
if (fk_name = assoc.primary_key_name) && col_names.include?(fk_name.to_s)
|
830
|
+
return fk_name
|
831
|
+
end
|
832
|
+
if (fk_name = assoc.inverse_of.primary_key_name) && col_names.include?(fk_name.to_s)
|
833
|
+
return fk_name
|
834
|
+
end
|
835
|
+
end
|
836
|
+
|
837
|
+
puts "* Wow, had no luck at all finding a foreign key for #{assoc.inspect}"
|
838
|
+
end
|
839
|
+
|
840
|
+
# unless build_tables.include?((foreign_class = tbs[2].class).underscore)
|
841
|
+
# build_table(build_tables, tbs[1].class, foreign_class, :has_one)
|
738
842
|
|
739
843
|
# Called before building any object linked through a has_one or has_many so that foreign key IDs
|
740
844
|
# can be added properly to those new objects. Finally at the end also called to save everything.
|
741
845
|
def self._save_pending(to_be_saved)
|
742
846
|
while (tbs = to_be_saved.pop)
|
743
847
|
# puts "Will be calling #{tbs[1].class.name} #{tbs[1]&.id} .#{tbs[2]} #{tbs[3].class.name} #{tbs[3]&.id}"
|
744
|
-
|
745
|
-
|
746
|
-
|
848
|
+
# Wire this one up if it had come from a has_one
|
849
|
+
if tbs[0] == tbs[1]
|
850
|
+
tbs[1].class.has_one(tbs[2]) unless tbs[1].respond_to?(tbs[2])
|
851
|
+
tbs[1].send(tbs[2], tbs[3])
|
852
|
+
end
|
747
853
|
|
748
854
|
ais = (tbs.first.class.respond_to?(:around_import_save) && tbs.first.class.method(:around_import_save)) ||
|
749
855
|
(respond_to?(:around_import_save) && method(:around_import_save))
|
@@ -764,7 +870,17 @@ module DutyFree
|
|
764
870
|
tbs[0] == tbs[1] || # From a has_one?
|
765
871
|
tbs.first.new_record?
|
766
872
|
|
767
|
-
tbs[1]
|
873
|
+
if tbs[1] == :has_and_belongs_to_many # also used by has_many :through associations
|
874
|
+
collection = tbs[3].send(tbs[2])
|
875
|
+
being_shoveled_id = tbs[0].send(tbs[0].class.primary_key)
|
876
|
+
if collection.empty? ||
|
877
|
+
!collection.pluck("#{(klass = collection.first.class).table_name}.#{klass.primary_key}").include?(being_shoveled_id)
|
878
|
+
collection << tbs[0]
|
879
|
+
# puts collection.inspect
|
880
|
+
end
|
881
|
+
else # Traditional belongs_to
|
882
|
+
tbs[1].send(tbs[2], tbs[3])
|
883
|
+
end
|
768
884
|
end
|
769
885
|
end
|
770
886
|
|
@@ -783,40 +899,71 @@ module DutyFree
|
|
783
899
|
template_detail_columns
|
784
900
|
end
|
785
901
|
|
786
|
-
# Recurse and return
|
787
|
-
# nested hashes to be used with ActiveRecord's .joins() to facilitate export
|
788
|
-
|
902
|
+
# Recurse and return three arrays -- one with all columns in sequence, and one a hierarchy of
|
903
|
+
# nested hashes to be used with ActiveRecord's .joins() to facilitate export, and finally
|
904
|
+
# one that lists tables that need to be built along the way.
|
905
|
+
def self._recurse_def(klass, cols, import_template, build_tables = nil, order_by = nil, assocs = [], joins = [], pre_prefix = '', prefix = '')
|
906
|
+
prefix = prefix[0..-2] if (is_build_table = prefix.end_with?('!'))
|
789
907
|
prefixes = ::DutyFree::Util._prefix_join([pre_prefix, prefix])
|
790
|
-
|
791
|
-
|
792
|
-
|
793
|
-
|
794
|
-
|
795
|
-
|
908
|
+
|
909
|
+
if prefix.present?
|
910
|
+
# An indication to build this table and model if it doesn't exist?
|
911
|
+
if is_build_table && build_tables
|
912
|
+
namespaces = prefix.split('::')
|
913
|
+
model_name = namespaces.map { |part| part.singularize.camelize }.join('::')
|
914
|
+
prefix = namespaces.last
|
915
|
+
# %%% If the model already exists, take belongs_to cues from it for building the table
|
916
|
+
if (is_need_model = Object.const_defined?(model_name)) &&
|
917
|
+
(is_need_table = ActiveRecord::ConnectionAdapters::SchemaStatements.table_exists?(
|
918
|
+
(path_name = ::DutyFree::Util._prefix_join([prefixes, prefix.pluralize]))
|
919
|
+
))
|
920
|
+
is_build_table = false
|
921
|
+
else
|
922
|
+
build_tables[path_name] = [namespaces, is_need_model, is_need_table] unless build_tables.include?(path_name)
|
923
|
+
end
|
924
|
+
end
|
925
|
+
unless is_build_table && build_tables
|
926
|
+
# Confirm we can actually navigate through this association
|
927
|
+
prefix_assoc = (assocs.last&.klass || klass).reflect_on_association(prefix) if prefix.present?
|
928
|
+
if prefix_assoc
|
929
|
+
assocs = assocs.dup << prefix_assoc
|
930
|
+
if order_by && [:has_many, :has_and_belongs_to_many].include?(prefix_assoc.macro) &&
|
931
|
+
(pk = prefix_assoc.active_record.primary_key)
|
932
|
+
order_by << ["#{prefixes.tr('.', '_')}_", pk]
|
933
|
+
end
|
934
|
+
end
|
796
935
|
end
|
797
936
|
end
|
798
|
-
|
937
|
+
cols = cols.inject([]) do |s, col|
|
799
938
|
s + if col.is_a?(Hash)
|
800
939
|
col.inject([]) do |s2, v|
|
801
|
-
|
802
|
-
|
940
|
+
if order_by
|
941
|
+
# Find what the type is for this guy
|
942
|
+
next_klass = (assocs.last&.klass || klass).reflect_on_association(v.first)&.klass
|
943
|
+
# Used to be: { v.first.to_sym => (joins_array = []) }
|
944
|
+
joins << { v.first.to_sym => (joins_array = [next_klass]) }
|
945
|
+
end
|
946
|
+
s2 + _recurse_def(klass, (v.last.is_a?(Array) ? v.last : [v.last]), import_template, build_tables, order_by, assocs, joins_array, prefixes, v.first.to_sym).first
|
803
947
|
end
|
804
948
|
elsif col.nil?
|
805
949
|
if assocs.empty?
|
806
950
|
[]
|
807
951
|
else
|
808
952
|
# Bring in from another class
|
809
|
-
|
953
|
+
# Used to be: { prefix => (joins_array = []) }
|
954
|
+
# %%% Probably need a next_klass thing like above
|
955
|
+
joins << { prefix => (joins_array = [klass]) } if order_by
|
810
956
|
# %%% Also bring in uniques and requireds
|
811
|
-
_recurse_def(klass, assocs.last.klass::IMPORT_TEMPLATE[:all], import_template, order_by, assocs, joins_array, prefixes).first
|
957
|
+
_recurse_def(klass, assocs.last.klass::IMPORT_TEMPLATE[:all], import_template, build_tables, order_by, assocs, joins_array, prefixes).first
|
812
958
|
end
|
813
959
|
else
|
814
960
|
[::DutyFree::Column.new(col, pre_prefix, prefix, assocs, klass, import_template[:as])]
|
815
961
|
end
|
816
962
|
end
|
817
|
-
[
|
963
|
+
[cols, joins]
|
818
964
|
end
|
819
965
|
end # module Extensions
|
966
|
+
# rubocop:enable Style/CommentedKeyword
|
820
967
|
|
821
968
|
# Rails < 4.0 doesn't have ActiveRecord::RecordNotUnique, so use the more generic ActiveRecord::ActiveRecordError instead
|
822
969
|
ar_not_unique_error = ActiveRecord.const_defined?('RecordNotUnique') ? ActiveRecord::RecordNotUnique : ActiveRecord::ActiveRecordError
|