duty_free 1.0.7 → 1.0.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,7 +6,11 @@ require 'duty_free/suggest_template'
6
6
 
7
7
  # :nodoc:
8
8
  module DutyFree
9
+ # rubocop:disable Style/CommentedKeyword
9
10
  module Extensions
11
+ MAX_ID = Arel.sql('MAX(id)')
12
+ IS_AMOEBA = Gem.loaded_specs['amoeba']
13
+
10
14
  def self.included(base)
11
15
  base.send :extend, ClassMethods
12
16
  base.send :extend, ::DutyFree::SuggestTemplate::ClassMethods
@@ -14,10 +18,6 @@ module DutyFree
14
18
 
15
19
  # :nodoc:
16
20
  module ClassMethods
17
- MAX_ID = Arel.sql('MAX(id)')
18
- # def self.extended(model)
19
- # end
20
-
21
21
  # Export at least column header, and optionally include all existing data as well
22
22
  def df_export(is_with_data = true, import_template = nil, use_inner_joins = false)
23
23
  use_inner_joins = true unless respond_to?(:left_joins)
@@ -53,8 +53,12 @@ module DutyFree
53
53
  if is_with_data
54
54
  order_by = []
55
55
  order_by << ['_', primary_key] if primary_key
56
+ all = import_template[:all] || import_template[:all!]
56
57
  # Automatically create a JOINs strategy and select list to get back all related rows
57
- template_cols, template_joins = ::DutyFree::Extensions._recurse_def(self, import_template[:all], import_template, order_by)
58
+ template_cols, template_joins = ::DutyFree::Extensions._recurse_def(self, all, import_template, nil, order_by)
59
+ # We do this so early here because it removes type objects from template_joins so then
60
+ # template_joins can immediately be used for inner and outer JOINs.
61
+ our_names = [[self, '_']] + ::DutyFree::Util._recurse_arel(template_joins)
58
62
  relation = use_inner_joins ? joins(template_joins) : left_joins(template_joins)
59
63
 
60
64
  # So we can properly create the SELECT list, create a mapping between our
@@ -71,12 +75,27 @@ module DutyFree
71
75
  # With AR < 3.2, "froms" brings up the top node, an Arel::Nodes::InnerJoin
72
76
  ::DutyFree::Util._recurse_arel(core.froms)
73
77
  end
74
- our_names = ['_'] + ::DutyFree::Util._recurse_arel(template_joins)
75
- mapping = our_names.zip(arel_alias_names).to_h
78
+ # Make sure our_names lines up with the arel_alias_name by comparing the ActiveRecord type.
79
+ # AR < 5.0 behaves differently than newer versions, and AR 5.0 and 5.1 have a bug in the
80
+ # way the types get determined, so if we want perfect results then we must compensate for
81
+ # these foibles. Thank goodness that AR 5.2 and later find the proper type and make it
82
+ # available through the type_caster object, which is what we use when building the list of
83
+ # arel_alias_names.
84
+ mapping = arel_alias_names.each_with_object({}) do |arel_alias_name, s|
85
+ if our_names.first&.first == arel_alias_name.first
86
+ s[our_names.first.last] = arel_alias_name.last
87
+ our_names.shift
88
+ end
89
+ s
90
+ end
76
91
  relation = (order_by.empty? ? relation : relation.order(order_by.map { |o| "#{mapping[o.first]}.#{o.last}" }))
77
92
  # puts mapping.inspect
78
93
  # puts relation.dup.select(template_cols.map { |x| x.to_s(mapping) }).to_sql
79
- relation.select(template_cols.map { |x| x.to_s(mapping) }).each do |result|
94
+
95
+ # Allow customisation of query before running it
96
+ relation = yield(relation, mapping) if block_given?
97
+
98
+ relation&.select(template_cols.map { |x| x.to_s(mapping) })&.each do |result|
80
99
  rows << ::DutyFree::Extensions._template_columns(self, import_template).map do |col|
81
100
  value = result.send(col)
82
101
  case value
@@ -93,399 +112,57 @@ module DutyFree
93
112
  rows
94
113
  end
95
114
 
96
- # With an array of incoming data, the first row having column names, perform the import
97
- def df_import(data, import_template = nil)
98
- instance_variable_set(:@defined_uniques, nil)
99
- instance_variable_set(:@valid_uniques, nil)
100
-
101
- import_template ||= if constants.include?(:IMPORT_TEMPLATE)
102
- self::IMPORT_TEMPLATE
103
- else
104
- suggest_template(0, false, false)
105
- end
106
- # puts "Chose #{import_template}"
107
- inserts = []
108
- updates = []
109
- counts = Hash.new { |h, k| h[k] = [] }
110
- errors = []
111
-
112
- is_first = true
113
- uniques = nil
114
- cols = nil
115
- starred = []
116
- partials = []
117
- all = import_template[:all]
118
- keepers = {}
119
- valid_unique = nil
120
- existing = {}
121
- devise_class = ''
122
- ret = nil
123
-
124
- # Multi-tenancy gem Apartment can be used if there are separate schemas per tenant
125
- reference_models = if Object.const_defined?('Apartment')
126
- Apartment.excluded_models
127
- else
128
- []
129
- end
130
-
131
- if Object.const_defined?('Devise')
132
- Object.const_get('Devise') # If this fails, devise_class will remain a blank string.
133
- devise_class = Devise.mappings.values.first.class_name
134
- reference_models -= [devise_class]
135
- else
136
- devise_class = ''
137
- end
138
-
139
- # Did they give us a filename?
140
- if data.is_a?(String)
141
- # Filenames with full paths can not be longer than 4096 characters, and can not
142
- # include newline characters
143
- data = if data.length <= 4096 && !data.index('\n')
144
- File.open(data)
145
- else
146
- # Any multi-line string is likely CSV data
147
- # %%% Test to see if TAB characters are present on the first line, instead of commas
148
- CSV.new(data)
149
- end
150
- end
151
- # Or perhaps us a file?
152
- if data.is_a?(File)
153
- # Use the "roo" gem if it's available
154
- data = if Object.const_defined?('Roo::Spreadsheet', { csv_options: { encoding: 'bom|utf-8' } })
155
- Roo::Spreadsheet.open(data)
156
- else
157
- # Otherwise generic CSV parsing
158
- require 'csv' unless Object.const_defined?('CSV')
159
- CSV.open(data)
160
- end
161
- end
162
-
163
- # Will show as just one transaction when using auditing solutions such as PaperTrail
164
- ActiveRecord::Base.transaction do
165
- # Check to see if they want to do anything before the whole import
166
- # First if defined in the import_template, then if there is a method in the class,
167
- # and finally (not yet implemented) a generic global before_import
168
- my_before_import = import_template[:before_import]
169
- my_before_import ||= respond_to?(:before_import) && method(:before_import)
170
- # my_before_import ||= some generic my_before_import
171
- if my_before_import
172
- last_arg_idx = my_before_import.parameters.length - 1
173
- arguments = [data, import_template][0..last_arg_idx]
174
- data = ret if (ret = my_before_import.call(*arguments)).is_a?(Enumerable)
175
- end
176
- data.each_with_index do |row, row_num|
177
- row_errors = {}
178
- if is_first # Anticipate that first row has column names
179
- uniques = import_template[:uniques]
180
-
181
- # Look for UTF-8 BOM in very first cell
182
- row[0] = row[0][3..-1] if row[0].start_with?([239, 187, 191].pack('U*'))
183
- # How about a first character of FEFF or FFFE to support UTF-16 BOMs?
184
- # FE FF big-endian (standard)
185
- # FF FE little-endian
186
- row[0] = row[0][1..-1] if [65_279, 65_534].include?(row[0][0].ord)
187
- cols = row.map { |col| (col || '').strip }
188
-
189
- # Unique column combinations can be called out explicitly in models using uniques: {...}, or to just
190
- # define one column at a time simply mark with an asterisk.
191
- # Track and clean up stars
192
- starred = cols.select do |col|
193
- if col[0] == '*'
194
- col.slice!(0)
195
- col.strip!
196
- end
197
- end
198
- partials = cols.select do |col|
199
- if col[0] == '~'
200
- col.slice!(0)
201
- col.strip!
202
- end
203
- end
204
- cols.map! { |col| ::DutyFree::Util._clean_name(col, import_template[:as]) }
205
- defined_uniques(uniques, cols, cols.join('|'), starred)
206
- # Make sure that at least half of them match what we know as being good column names
207
- template_column_objects = ::DutyFree::Extensions._recurse_def(self, import_template[:all], import_template).first
208
- cols.each_with_index do |col, idx|
209
- # prefixes = col_detail.pre_prefix + (col_detail.prefix.blank? ? [] : [col_detail.prefix])
210
- keepers[idx] = template_column_objects.find { |col_obj| col_obj.titleize == col }
211
- # puts "Could not find a match for column #{idx + 1}, #{col}" if keepers[idx].nil?
212
- end
213
- raise ::DutyFree::LessThanHalfAreMatchingColumnsError, I18n.t('import.altered_import_template_coumns') if keepers.length < (cols.length / 2) - 1
214
-
215
- # Returns just the first valid unique lookup set if there are multiple
216
- valid_unique = find_existing(uniques, cols, starred, import_template, keepers, false)
217
- # Make a lookup from unique values to specific IDs
218
- existing = pluck(*([:id] + valid_unique.keys)).each_with_object(existing) do |v, s|
219
- s[v[1..-1].map(&:to_s)] = v.first
220
- s
221
- end
222
- is_first = false
223
- else # Normal row of data
224
- is_insert = false
225
- existing_unique = valid_unique.inject([]) do |s, v|
226
- s << if v.last.is_a?(Array)
227
- v.last[0].where(v.last[1] => row[v.last[2]]).limit(1).pluck(MAX_ID).first.to_s
228
- else
229
- row[v.last].to_s
230
- end
231
- end
232
- to_be_saved = []
233
- # Check to see if they want to preprocess anything
234
- existing_unique = @before_process.call(valid_unique, existing_unique) if @before_process ||= import_template[:before_process]
235
- if (criteria = existing[existing_unique])
236
- obj = find(criteria)
237
- else
238
- is_insert = true
239
- to_be_saved << [obj = new]
240
- end
241
- sub_obj = nil
242
- is_has_one = false
243
- has_ones = []
244
- polymorphics = []
245
- sub_objects = {}
246
- this_path = nil
247
- keepers.each do |key, v|
248
- next if v.nil?
249
-
250
- sub_obj = obj
251
- this_path = +''
252
- # puts "p: #{v.path}"
253
- v.path.each_with_index do |path_part, idx|
254
- this_path << (this_path.blank? ? path_part.to_s : ",#{path_part}")
255
- unless (sub_next = sub_objects[this_path])
256
- # Check if we're hitting reference data / a lookup thing
257
- assoc = v.prefix_assocs[idx]
258
- # belongs_to some lookup (reference) data
259
- if assoc && reference_models.include?(assoc.class_name)
260
- lookup_match = assoc.klass.find_by(v.name => row[key])
261
- # Do a partial match if this column allows for it
262
- # and we only find one matching result.
263
- if lookup_match.nil? && partials.include?(v.titleize)
264
- lookup_match ||= assoc.klass.where("#{v.name} LIKE '#{row[key]}%'")
265
- lookup_match = (lookup_match.length == 1 ? lookup_match.first : nil)
266
- end
267
- sub_obj.send("#{path_part}=", lookup_match) unless lookup_match.nil?
268
- # Reference data from the public level means we stop here
269
- sub_obj = nil
270
- break
271
- end
272
- # Get existing related object, or create a new one
273
- # This first part works for belongs_to. has_many and has_one get sorted below.
274
- if (sub_next = sub_obj.send(path_part)).nil? && assoc.belongs_to?
275
- klass = Object.const_get(assoc&.class_name)
276
- # Try to find a unique item if one is referenced
277
- sub_next = nil
278
- begin
279
- trim_prefix = v.titleize[0..-(v.name.length + 2)]
280
- trim_prefix << ' ' unless trim_prefix.blank?
281
- sub_next, criteria = klass.find_existing(uniques, cols, starred, import_template, keepers, nil, row, klass, all, trim_prefix)
282
- rescue ::DutyFree::NoUniqueColumnError
283
- end
284
- # puts "#{v.path} #{criteria.inspect}"
285
- bt_name = "#{path_part}="
286
- unless sub_next || (klass == sub_obj.class && criteria.empty?)
287
- sub_next = klass.new(criteria || {})
288
- to_be_saved << [sub_next, sub_obj, bt_name, sub_next]
289
- end
290
- # This wires it up in memory, but doesn't yet put the proper foreign key ID in
291
- # place when the primary object is a new one (and thus not having yet been saved
292
- # doesn't yet have an ID).
293
- # binding.pry if !sub_next.new_record? && sub_next.name == 'Squidget Widget' # !sub_obj.changed?
294
- # # %%% Question is -- does the above set changed? when a foreign key is not yet set
295
- # # and only the in-memory object has changed?
296
- is_yet_changed = sub_obj.changed?
297
- sub_obj.send(bt_name, sub_next)
298
-
299
- # We need this in the case of updating the primary object across a belongs_to when
300
- # the foreign one already exists and is not otherwise changing, such as when it is
301
- # not a new one, so wouldn't otherwise be getting saved.
302
- to_be_saved << [sub_obj] if !sub_obj.new_record? && !is_yet_changed && sub_obj.changed?
303
- # From a has_many or has_one?
304
- # Rails 4.0 and later can do: sub_next.is_a?(ActiveRecord::Associations::CollectionProxy)
305
- elsif [:has_many, :has_one].include?(assoc.macro) && !assoc.options[:through]
306
- ::DutyFree::Extensions._save_pending(to_be_saved)
307
- # Try to find a unique item if one is referenced
308
- # %%% There is possibility that when bringing in related classes using a nil
309
- # in IMPORT_TEMPLATE[:all] that this will break. Need to test deeply nested things.
310
- start = (v.pre_prefix.blank? ? 0 : v.pre_prefix.length)
311
- trim_prefix = v.titleize[start..-(v.name.length + 2)]
312
- trim_prefix << ' ' unless trim_prefix.blank?
313
- # assoc.inverse_of is the belongs_to side of the has_many train we came in here on.
314
- sub_hm, criteria = assoc.klass.find_existing(uniques, cols, starred, import_template, keepers, assoc.inverse_of, row, sub_next, all, trim_prefix)
315
- # If still not found then create a new related object using this has_many collection
316
- # (criteria.empty? ? nil : sub_next.new(criteria))
317
- if sub_hm
318
- sub_next = sub_hm
319
- elsif assoc.macro == :has_one
320
- # assoc.active_record.name.underscore is only there to support older Rails
321
- # that doesn't do automatic inverse_of
322
- ho_name = "#{assoc.inverse_of&.name || assoc.active_record.name.underscore}="
323
- sub_next = assoc.klass.new(criteria)
324
- to_be_saved << [sub_next, sub_next, ho_name, sub_obj]
325
- else
326
- # Two other methods that are possible to check for here are :conditions and
327
- # :sanitized_conditions, which do not exist in Rails 4.0 and later.
328
- sub_next = if assoc.respond_to?(:require_association)
329
- # With Rails < 4.0, sub_next could come in as an Array or a broken CollectionProxy
330
- assoc.klass.new({ fk_from(assoc) => sub_obj.send(sub_obj.class.primary_key) }.merge(criteria))
331
- else
332
- sub_next.proxy_association.reflection.instance_variable_set(:@foreign_key, fk_from(assoc))
333
- sub_next.new(criteria)
334
- end
335
- to_be_saved << [sub_next]
336
- end
337
- end
338
- # Look for possible missing polymorphic detail
339
- # Maybe can test for this via assoc.through_reflection
340
- if assoc.is_a?(ActiveRecord::Reflection::ThroughReflection) &&
341
- (delegate = assoc.send(:delegate_reflection)&.active_record&.reflect_on_association(assoc.source_reflection_name)) &&
342
- delegate.options[:polymorphic]
343
- polymorphics << { parent: sub_next, child: sub_obj, type_col: delegate.foreign_type, id_col: delegate.foreign_key.to_s }
344
- end
345
- unless sub_next.nil?
346
- # if sub_next.class.name == devise_class && # only for Devise users
347
- # sub_next.email =~ Devise.email_regexp
348
- # if existing.include?([sub_next.email])
349
- # User already exists
350
- # else
351
- # sub_next.invite!
352
- # end
353
- # end
354
- sub_objects[this_path] = sub_next if this_path.present?
355
- end
356
- end
357
- sub_obj = sub_next
358
- end
359
- next if sub_obj.nil?
115
+ def df_import(data, import_template = nil, insert_only = false)
116
+ ::DutyFree::Extensions.import(self, data, import_template, insert_only)
117
+ end
360
118
 
361
- next unless sub_obj.respond_to?(sym = "#{v.name}=".to_sym)
119
+ private
362
120
 
363
- col_type = sub_obj.class.columns_hash[v.name.to_s]&.type
364
- if col_type.nil? && (virtual_columns = import_template[:virtual_columns]) &&
365
- (virtual_columns = virtual_columns[this_path] || virtual_columns)
366
- col_type = virtual_columns[v.name]
367
- end
368
- if col_type == :boolean
369
- if row[key].nil?
370
- # Do nothing when it's nil
371
- elsif %w[true t yes y].include?(row[key]&.strip&.downcase) # Used to cover 'on'
372
- row[key] = true
373
- elsif %w[false f no n].include?(row[key]&.strip&.downcase) # Used to cover 'off'
374
- row[key] = false
375
- else
376
- row_errors[v.name] ||= []
377
- row_errors[v.name] << "Boolean value \"#{row[key]}\" in column #{key + 1} not recognized"
378
- end
379
- end
121
+ def _defined_uniques(uniques, cols = [], col_list = nil, starred = [], trim_prefix = '')
122
+ col_list ||= cols.join('|')
123
+ unless (defined_uniq = (@defined_uniques ||= {})[col_list])
124
+ utilised = {} # Track columns that have been referenced thusfar
125
+ defined_uniq = uniques.each_with_object({}) do |unique, s|
126
+ if unique.is_a?(Array)
127
+ key = []
128
+ value = []
129
+ unique.each do |unique_part|
130
+ val = (unique_part_name = unique_part.to_s.titleize).start_with?(trim_prefix) &&
131
+ cols.index(upn = unique_part_name[trim_prefix.length..-1])
132
+ next unless val
380
133
 
381
- is_yet_changed = sub_obj.changed?
382
- sub_obj.send(sym, row[key])
383
- # If this one is transitioning from having not changed to now having been changed,
384
- # and is not a new one anyway that would already be lined up to get saved, then we
385
- # mark it to now be saved.
386
- to_be_saved << [sub_obj] if !sub_obj.new_record? && !is_yet_changed && sub_obj.changed?
387
- # else
388
- # puts " #{sub_obj.class.name} doesn't respond to #{sym}"
134
+ key << upn
135
+ value << val
389
136
  end
390
- # Try to save final sub-object(s) if any exist
391
- ::DutyFree::Extensions._save_pending(to_be_saved)
392
-
393
- # Reinstate any missing polymorphic _type and _id values
394
- polymorphics.each do |poly|
395
- if !poly[:parent].new_record? || poly[:parent].save
396
- poly[:child].send("#{poly[:type_col]}=".to_sym, poly[:parent].class.name)
397
- poly[:child].send("#{poly[:id_col]}=".to_sym, poly[:parent].id)
398
- end
137
+ unless key.empty?
138
+ s[key] = value
139
+ utilised[key] = nil
399
140
  end
400
-
401
- # Give a window of opportunity to tweak user objects controlled by Devise
402
- obj_class = obj.class
403
- is_do_save = if obj_class.respond_to?(:before_devise_save) && obj_class.name == devise_class
404
- obj_class.before_devise_save(obj, existing)
405
- else
406
- true
407
- end
408
-
409
- if obj.valid?
410
- obj.save if is_do_save
411
- # Snag back any changes to the unique columns. (For instance, Devise downcases email addresses.)
412
- existing_unique = valid_unique.keys.inject([]) { |s, v| s << obj.send(v).to_s }
413
- # Update the duplicate counts and inserted / updated results
414
- counts[existing_unique] << row_num
415
- (is_insert ? inserts : updates) << { row_num => existing_unique } if is_do_save
416
- # Track this new object so we can properly sense any duplicates later
417
- existing[existing_unique] = obj.id
418
- else
419
- row_errors.merge! obj.errors.messages
141
+ else
142
+ val = (unique_name = unique.to_s.titleize).start_with?(trim_prefix) &&
143
+ cols.index(un = unique_name[trim_prefix.length..-1])
144
+ if val
145
+ s[[un]] = [val]
146
+ utilised[[un]] = nil
420
147
  end
421
- errors << { row_num => row_errors } unless row_errors.empty?
422
148
  end
423
- end
424
- duplicates = counts.each_with_object([]) do |v, s|
425
- s += v.last[1..-1].map { |line_num| { line_num => v.first } } if v.last.count > 1
426
149
  s
427
150
  end
428
- ret = { inserted: inserts, updated: updates, duplicates: duplicates, errors: errors }
429
-
430
- # Check to see if they want to do anything after the import
431
- # First if defined in the import_template, then if there is a method in the class,
432
- # and finally (not yet implemented) a generic global after_import
433
- my_after_import = import_template[:after_import]
434
- my_after_import ||= respond_to?(:after_import) && method(:after_import)
435
- # my_after_import ||= some generic my_after_import
436
- if my_after_import
437
- last_arg_idx = my_after_import.parameters.length - 1
438
- arguments = [ret][0..last_arg_idx]
439
- ret = ret2 if (ret2 = my_after_import.call(*arguments)).is_a?(Hash)
440
- end
441
- end
442
- ret
443
- end
444
-
445
- def fk_from(assoc)
446
- # Try first to trust whatever they've marked as being the foreign_key, and then look
447
- # at the inverse's foreign key setting if available. In all cases don't accept
448
- # anything that's not backed with a real column in the table.
449
- col_names = assoc.klass.column_names
450
- if (fk_name = assoc.options[:foreign_key]) && col_names.include?(fk_name.to_s)
451
- return fk_name
452
- end
453
- if (fk_name = assoc.inverse_of&.options&.fetch(:foreign_key) { nil }) &&
454
- col_names.include?(fk_name.to_s)
455
- return fk_name
456
- end
457
- if assoc.respond_to?(:foreign_key) && (fk_name = assoc.foreign_key) &&
458
- col_names.include?(fk_name.to_s)
459
- return fk_name
460
- end
461
- if (fk_name = assoc.inverse_of&.foreign_key) &&
462
- col_names.include?(fk_name.to_s)
463
- return fk_name
464
- end
465
- if (fk_name = assoc.inverse_of&.association_foreign_key) &&
466
- col_names.include?(fk_name.to_s)
467
- return fk_name
468
- end
469
- # Don't let this fool you -- we really are in search of the foreign key name here,
470
- # and Rails 3.0 and older used some fairly interesting conventions, calling it instead
471
- # the "primary_key_name"!
472
- if assoc.respond_to?(:primary_key_name)
473
- if (fk_name = assoc.primary_key_name) && col_names.include?(fk_name.to_s)
474
- return fk_name
475
- end
476
- if (fk_name = assoc.inverse_of.primary_key_name) && col_names.include?(fk_name.to_s)
477
- return fk_name
151
+ if defined_uniq.empty?
152
+ (starred - utilised.keys).each { |star| defined_uniq[[star]] = [cols.index(star)] }
153
+ # %%% puts "Tried to establish #{defined_uniq.inspect}"
478
154
  end
155
+ @defined_uniques[col_list] = defined_uniq
479
156
  end
480
-
481
- puts "* Wow, had no luck at all finding a foreign key for #{assoc.inspect}"
157
+ defined_uniq
482
158
  end
483
159
 
484
160
  # For use with importing, based on the provided column list calculate all valid combinations
485
161
  # of unique columns. If there is no valid combination, throws an error.
486
- # Returns an object found by this means.
487
- def find_existing(uniques, cols, starred, import_template, keepers, train_we_came_in_here_on,
488
- row = nil, klass_or_collection = nil, template_all = nil, trim_prefix = '')
162
+ # Returns an object found by this means, as well as the criteria that was used to find it.
163
+ def _find_existing(uniques, cols, starred, import_template, keepers, train_we_came_in_here_on, insert_only,
164
+ row = nil, klass_or_collection = nil, template_all = nil, trim_prefix = '',
165
+ assoc = nil, base_obj = nil)
489
166
  unless trim_prefix.blank?
490
167
  cols = cols.map { |c| c.start_with?(trim_prefix) ? c[trim_prefix.length..-1] : nil }
491
168
  starred = starred.each_with_object([]) do |v, s|
@@ -536,7 +213,7 @@ module DutyFree
536
213
  # folks based on having their email as a unique identifier, and other folks by having a
537
214
  # combination of their name and street address as unique, and use both of those possible
538
215
  # unique variations to update phone numbers, and do that all as a part of one import.)
539
- all_vus = defined_uniques(uniques, cols, col_list, starred, trim_prefix)
216
+ all_vus = _defined_uniques(uniques, cols, col_list, starred, trim_prefix)
540
217
 
541
218
  # %%% Ultimately may consider making this recursive
542
219
  reflect_on_all_associations.each do |sn_bt|
@@ -586,7 +263,8 @@ module DutyFree
586
263
  # If we're processing a row then this list of foreign key column name entries, named such as
587
264
  # "order_id" or "product_id" instead of column-specific stuff like "Order Date" and "Product Name",
588
265
  # is kept until the last and then gets merged on top of the other criteria before being returned.
589
- bt_criteria[(fk_name = sn_bt.foreign_key)] = fk_id
266
+ fk_name = sn_bt.foreign_key
267
+ bt_criteria[fk_name] = fk_id unless bt_criteria.include?(fk_name)
590
268
 
591
269
  # Check to see if belongs_tos are generally required on this specific table
592
270
  bt_req_by_default = sn_bt.klass.respond_to?(:belongs_to_required_by_default) &&
@@ -606,7 +284,7 @@ module DutyFree
606
284
  (sn_bt.options[:optional] || !bt_req_by_default)
607
285
 
608
286
  # Add to the criteria
609
- criteria[fk_name] = fk_id
287
+ criteria[fk_name] = fk_id if insert_only && !criteria.include?(fk_name)
610
288
  end
611
289
  end
612
290
 
@@ -656,19 +334,63 @@ module DutyFree
656
334
  # Find by all corresponding columns
657
335
  if (row_value = row[v])
658
336
  new_criteria_all_nil = false
659
- criteria[k_sym] = row_value # The data, or how to look up the data
337
+ criteria[k_sym] = row_value # The data
660
338
  end
661
339
  end
662
340
  end
341
+ # puts uniq_lookups.inspect
663
342
 
664
- return uniq_lookups.merge(criteria) if only_valid_uniques
665
-
343
+ return [uniq_lookups.merge(criteria), bt_criteria] if only_valid_uniques
666
344
  # If there's nothing to match upon then we're out
667
- return [nil, {}] if new_criteria_all_nil
345
+ return [nil, {}, {}] if new_criteria_all_nil
668
346
 
669
347
  # With this criteria, find any matching has_many row we can so we can update it.
670
348
  # First try directly looking it up through ActiveRecord.
671
- found_object = klass_or_collection.find_by(criteria)
349
+ klass_or_collection ||= self # Comes in as nil for has_one with no object yet attached
350
+ # HABTM proxy
351
+ # binding.pry if klass_or_collection.is_a?(ActiveRecord::Associations::CollectionProxy)
352
+ # if klass_or_collection.respond_to?(:proxy_association) && klass_or_collection.proxy_association.options.include?(:through)
353
+ # klass_or_collection.proxy_association.association_scope.to_a
354
+
355
+ # if assoc.respond_to?(:require_association) && klass_or_collection.is_a?(Array)
356
+ # # else
357
+ # # klass_or_collection = assoc.klass
358
+ # end
359
+ # end
360
+ found_object = case klass_or_collection
361
+ when ActiveRecord::Base # object from a has_one?
362
+ existing_object = klass_or_collection
363
+ klass_or_collection = klass_or_collection.class
364
+ other_object = klass_or_collection.find_by(criteria)
365
+ pk = klass_or_collection.primary_key
366
+ existing_object.send(pk) == other_object&.send(pk) ? existing_object : other_object
367
+ when Array # has_* in AR < 4.0
368
+ # Old AR doesn't have a CollectionProxy that can do any of this on its own.
369
+ base_id = base_obj.send(base_obj.class.primary_key)
370
+ if assoc.macro == :has_and_belongs_to_many || (assoc.macro == :has_many && assoc.options[:through])
371
+ # Find all association foreign keys, then find or create the foreign object
372
+ # based on criteria, and finally put an entry with both foreign keys into
373
+ # the associative table unless it already exists.
374
+ ajt = assoc.through_reflection&.table_name || assoc.join_table
375
+ fk = assoc.foreign_key
376
+ afk = assoc.association_foreign_key
377
+ existing_ids = ActiveRecord::Base.connection.execute(
378
+ "SELECT #{afk} FROM #{ajt} WHERE #{fk} = #{base_id}"
379
+ ).map { |r| r[afk] }
380
+ new_or_existing = assoc.klass.find_or_create_by(criteria)
381
+ new_or_existing_id = new_or_existing.send(new_or_existing.class.primary_key)
382
+ unless existing_ids.include?(new_or_existing_id)
383
+ ActiveRecord::Base.connection.execute(
384
+ "INSERT INTO #{ajt} (#{fk}, #{afk}) VALUES (#{base_id}, #{new_or_existing_id})"
385
+ )
386
+ end
387
+ new_or_existing
388
+ else # Must be a has_many
389
+ assoc.klass.find_or_create_by(criteria.merge({ assoc.foreign_key => base_id }))
390
+ end
391
+ else
392
+ klass_or_collection.find_by(criteria)
393
+ end
672
394
  # If not successful, such as when fields are exposed via helper methods instead of being
673
395
  # real columns in the database tables, try this more intensive approach. This is useful
674
396
  # if you had full name kind of data coming in on a spreadsheeet, but in the destination
@@ -691,59 +413,453 @@ module DutyFree
691
413
  # Standard criteria as well as foreign key column name detail with exact foreign keys
692
414
  # that match up to a primary key so that if needed a new related object can be built,
693
415
  # complete with all its association detail.
694
- [found_object, criteria.merge(bt_criteria)]
416
+ [found_object, criteria, bt_criteria]
417
+ end # _find_existing
418
+ end # module ClassMethods
419
+
420
+ # With an array of incoming data, the first row having column names, perform the import
421
+ def self.import(obj_klass, data, import_template = nil, insert_only)
422
+ instance_variable_set(:@defined_uniques, nil)
423
+ instance_variable_set(:@valid_uniques, nil)
424
+
425
+ import_template ||= if obj_klass.constants.include?(:IMPORT_TEMPLATE)
426
+ obj_klass::IMPORT_TEMPLATE
427
+ else
428
+ obj_klass.suggest_template(0, false, false)
429
+ end
430
+ # puts "Chose #{import_template}"
431
+ inserts = []
432
+ updates = []
433
+ counts = Hash.new { |h, k| h[k] = [] }
434
+ errors = []
435
+
436
+ is_first = true
437
+ uniques = nil
438
+ cols = nil
439
+ starred = []
440
+ partials = []
441
+ # See if we can find the model if given only a string
442
+ if obj_klass.is_a?(String)
443
+ obj_klass_camelized = obj_klass.camelize.singularize
444
+ obj_klass = Object.const_get(obj_klass_camelized) if Object.const_defined?(obj_klass_camelized)
445
+ end
446
+ table_name = obj_klass.table_name unless obj_klass.is_a?(String)
447
+ is_build_table = if import_template.include?(:all!)
448
+ # Search for presence of this table
449
+ table_name = obj_klass.underscore.pluralize if obj_klass.is_a?(String)
450
+ !ActiveRecord::ConnectionAdapters::SchemaStatements.table_exists?(table_name)
451
+ else
452
+ false
453
+ end
454
+ # If we do need to build the table then we require an :all!, otherwise in the case that the table
455
+ # does not need to be built or was already built then try :all, and fall back on :all!.
456
+ all = import_template[is_build_table ? :all! : :all] || import_template[:all!]
457
+ keepers = {}
458
+ valid_unique = nil
459
+ existing = {}
460
+ devise_class = ''
461
+ ret = nil
462
+
463
+ # Multi-tenancy gem Apartment can be used if there are separate schemas per tenant
464
+ reference_models = if Object.const_defined?('Apartment')
465
+ Apartment.excluded_models
466
+ else
467
+ []
468
+ end
469
+
470
+ if Object.const_defined?('Devise')
471
+ Object.const_get('Devise') # If this fails, devise_class will remain a blank string.
472
+ devise_class = Devise.mappings.values.first.class_name
473
+ reference_models -= [devise_class]
474
+ else
475
+ devise_class = ''
695
476
  end
696
477
 
697
- private
478
+ # Did they give us a filename?
479
+ if data.is_a?(String)
480
+ # Filenames with full paths can not be longer than 4096 characters, and can not
481
+ # include newline characters
482
+ data = if data.length <= 4096 && !data.index('\n')
483
+ File.open(data)
484
+ else
485
+ # Any multi-line string is likely CSV data
486
+ # %%% Test to see if TAB characters are present on the first line, instead of commas
487
+ CSV.new(data)
488
+ end
489
+ end
490
+ # Or perhaps us a file?
491
+ if data.is_a?(File)
492
+ # Use the "roo" gem if it's available
493
+ # When we're ready to try parsing this thing on our own, shared strings and all, then use
494
+ # the rubyzip gem also along with it:
495
+ # https://github.com/rubyzip/rubyzip
496
+ # require 'zip'
497
+ data = if Object.const_defined?('Roo::Spreadsheet', { csv_options: { encoding: 'bom|utf-8' } })
498
+ Roo::Spreadsheet.open(data)
499
+ else
500
+ # Otherwise generic CSV parsing
501
+ require 'csv' unless Object.const_defined?('CSV')
502
+ CSV.open(data)
503
+ end
504
+ end
698
505
 
699
- def defined_uniques(uniques, cols = [], col_list = nil, starred = [], trim_prefix = '')
700
- col_list ||= cols.join('|')
701
- unless (defined_uniq = (@defined_uniques ||= {})[col_list])
702
- utilised = {} # Track columns that have been referenced thusfar
703
- defined_uniq = uniques.each_with_object({}) do |unique, s|
704
- if unique.is_a?(Array)
705
- key = []
706
- value = []
707
- unique.each do |unique_part|
708
- val = (unique_part_name = unique_part.to_s.titleize).start_with?(trim_prefix) &&
709
- cols.index(upn = unique_part_name[trim_prefix.length..-1])
710
- next unless val
506
+ # Will show as just one transaction when using auditing solutions such as PaperTrail
507
+ ActiveRecord::Base.transaction do
508
+ # Check to see if they want to do anything before the whole import
509
+ # First if defined in the import_template, then if there is a method in the class,
510
+ # and finally (not yet implemented) a generic global before_import
511
+ my_before_import = import_template[:before_import]
512
+ my_before_import ||= respond_to?(:before_import) && method(:before_import)
513
+ # my_before_import ||= some generic my_before_import
514
+ if my_before_import
515
+ last_arg_idx = my_before_import.parameters.length - 1
516
+ arguments = [data, import_template][0..last_arg_idx]
517
+ data = ret if (ret = my_before_import.call(*arguments)).is_a?(Enumerable)
518
+ end
519
+ build_tables = nil
520
+ data.each_with_index do |row, row_num|
521
+ row_errors = {}
522
+ if is_first # Anticipate that first row has column names
523
+ uniques = import_template[:uniques]
524
+
525
+ # Look for UTF-8 BOM in very first cell
526
+ row[0] = row[0][3..-1] if row[0].start_with?([239, 187, 191].pack('U*'))
527
+ # How about a first character of FEFF or FFFE to support UTF-16 BOMs?
528
+ # FE FF big-endian (standard)
529
+ # FF FE little-endian
530
+ row[0] = row[0][1..-1] if [65_279, 65_534].include?(row[0][0].ord)
531
+ cols = row.map { |col| (col || '').strip }
532
+
533
+ # Unique column combinations can be called out explicitly in models using uniques: {...}, or to just
534
+ # define one column at a time simply mark with an asterisk.
535
+ # Track and clean up stars
536
+ starred = cols.select do |col|
537
+ if col[0] == '*'
538
+ col.slice!(0)
539
+ col.strip!
540
+ end
541
+ end
542
+ partials = cols.select do |col|
543
+ if col[0] == '~'
544
+ col.slice!(0)
545
+ col.strip!
546
+ end
547
+ end
548
+ cols.map! { |col| ::DutyFree::Util._clean_name(col, import_template[:as]) }
549
+ obj_klass.send(:_defined_uniques, uniques, cols, cols.join('|'), starred)
550
+ # Main object asking for table to be built?
551
+ build_tables = {}
552
+ build_tables[path_name] = [namespaces, is_need_model, is_need_table] if is_build_table
553
+ # Make sure that at least half of them match what we know as being good column names
554
+ template_column_objects = ::DutyFree::Extensions._recurse_def(obj_klass, import_template[:all], import_template, build_tables).first
555
+ cols.each_with_index do |col, idx|
556
+ # prefixes = col_detail.pre_prefix + (col_detail.prefix.blank? ? [] : [col_detail.prefix])
557
+ # %%% Would be great here if when one comes back nil, try to find the closest match
558
+ # and indicate to the user a "did you mean?" about it.
559
+ keepers[idx] = template_column_objects.find { |col_obj| col_obj.titleize == col }
560
+ # puts "Could not find a match for column #{idx + 1}, #{col}" if keepers[idx].nil?
561
+ end
562
+ raise ::DutyFree::LessThanHalfAreMatchingColumnsError, I18n.t('import.altered_import_template_coumns') if keepers.length < (cols.length / 2) - 1
563
+
564
+ # Returns just the first valid unique lookup set if there are multiple
565
+ valid_unique, bt_criteria = obj_klass.send(:_find_existing, uniques, cols, starred, import_template, keepers, false, insert_only)
566
+ # Make a lookup from unique values to specific IDs
567
+ existing = obj_klass.pluck(*([:id] + valid_unique.keys)).each_with_object(existing) do |v, s|
568
+ s[v[1..-1].map(&:to_s)] = v.first
569
+ s
570
+ end
571
+ is_first = false
572
+ else # Normal row of data
573
+ is_insert = false
574
+ existing_unique = valid_unique.inject([]) do |s, v|
575
+ s << if v.last.is_a?(Array)
576
+ v.last[0].where(v.last[1] => row[v.last[2]]).limit(1).pluck(MAX_ID).first.to_s
577
+ else
578
+ row[v.last].to_s
579
+ end
580
+ end
581
+ to_be_saved = []
582
+ # Check to see if they want to preprocess anything
583
+ existing_unique = @before_process.call(valid_unique, existing_unique) if @before_process ||= import_template[:before_process]
584
+ obj = if (criteria = existing[existing_unique])
585
+ obj_klass.find(criteria)
586
+ else
587
+ is_insert = true
588
+ # unless build_tables.empty? # include?()
589
+ # binding.pry
590
+ # x = 5
591
+ # end
592
+ obj_klass.new
593
+ end
594
+ to_be_saved << [obj] unless criteria # || this one has any belongs_to that will be modified here
595
+ sub_obj = nil
596
+ polymorphics = []
597
+ sub_objects = {}
598
+ this_path = nil
599
+ keepers.each do |key, v|
600
+ next if v.nil?
601
+
602
+ sub_obj = obj
603
+ this_path = +''
604
+ # puts "p: #{v.path}"
605
+ v.path.each_with_index do |path_part, idx|
606
+ this_path << (this_path.blank? ? path_part.to_s : ",#{path_part}")
607
+ unless (sub_next = sub_objects[this_path])
608
+ # Check if we're hitting reference data / a lookup thing
609
+ assoc = v.prefix_assocs[idx]
610
+ # belongs_to some lookup (reference) data
611
+ if assoc && reference_models.include?(assoc.class_name)
612
+ lookup_match = assoc.klass.find_by(v.name => row[key])
613
+ # Do a partial match if this column allows for it
614
+ # and we only find one matching result.
615
+ if lookup_match.nil? && partials.include?(v.titleize)
616
+ lookup_match ||= assoc.klass.where("#{v.name} LIKE '#{row[key]}%'")
617
+ lookup_match = (lookup_match.length == 1 ? lookup_match.first : nil)
618
+ end
619
+ sub_obj.send("#{path_part}=", lookup_match) unless lookup_match.nil?
620
+ # Reference data from the public level means we stop here
621
+ sub_obj = nil
622
+ break
623
+ end
624
+ # Get existing related object, or create a new one
625
+ # This first part works for belongs_to. has_many and has_one get sorted below.
626
+ # start = (v.pre_prefix.blank? ? 0 : v.pre_prefix.length)
627
+ start = 0
628
+ trim_prefix = v.titleize[start..-(v.name.length + 2)]
629
+ trim_prefix << ' ' unless trim_prefix.blank?
630
+ binding.pry unless assoc
631
+ if assoc.belongs_to?
632
+ klass = Object.const_get(assoc&.class_name)
633
+ # Try to find a unique item if one is referenced
634
+ sub_next = nil
635
+ begin
636
+ sub_next, criteria, bt_criteria = klass.send(:_find_existing, uniques, cols, starred, import_template, keepers, nil,
637
+ false, # insert_only
638
+ row, klass, all, trim_prefix, assoc)
639
+ rescue ::DutyFree::NoUniqueColumnError
640
+ end
641
+ bt_name = "#{path_part}="
642
+ # Not yet wired up to the right one, or going to the parent of a self-referencing model?
643
+ # puts "#{v.path} #{criteria.inspect}"
644
+ unless sub_next || (klass == sub_obj.class && (all_criteria = criteria.merge(bt_criteria)).empty?)
645
+ sub_next = klass.new(all_criteria || {})
646
+ to_be_saved << [sub_next, sub_obj, bt_name, sub_next]
647
+ end
648
+ # This wires it up in memory, but doesn't yet put the proper foreign key ID in
649
+ # place when the primary object is a new one (and thus not having yet been saved
650
+ # doesn't yet have an ID).
651
+ # binding.pry if !sub_next.new_record? && sub_next.name == 'Squidget Widget' # !sub_obj.changed?
652
+ # # %%% Question is -- does the above set changed? when a foreign key is not yet set
653
+ # # and only the in-memory object has changed?
654
+ is_yet_changed = sub_obj.changed?
655
+ sub_obj.send(bt_name, sub_next)
656
+
657
+ # We need this in the case of updating the primary object across a belongs_to when
658
+ # the foreign one already exists and is not otherwise changing, such as when it is
659
+ # not a new one, so wouldn't otherwise be getting saved.
660
+ to_be_saved << [sub_obj] if !sub_obj.new_record? && !is_yet_changed && sub_obj.changed?
661
+ # From a has_many or has_one?
662
+ # Rails 4.0 and later can do: sub_next.is_a?(ActiveRecord::Associations::CollectionProxy)
663
+ elsif [:has_many, :has_one, :has_and_belongs_to_many].include?(assoc.macro) # && !assoc.options[:through]
664
+ ::DutyFree::Extensions._save_pending(to_be_saved)
665
+ sub_next = sub_obj.send(path_part)
666
+ # Try to find a unique item if one is referenced
667
+ # %%% There is possibility that when bringing in related classes using a nil
668
+ # in IMPORT_TEMPLATE[:all] that this will break. Need to test deeply nested things.
669
+
670
+ # assoc.inverse_of is the belongs_to side of the has_many train we came in here on.
671
+ sub_hm, criteria, bt_criteria = assoc.klass.send(:_find_existing, uniques, cols, starred, import_template, keepers, assoc.inverse_of,
672
+ false, # insert_only
673
+ row, sub_next, all, trim_prefix, assoc,
674
+ # Just in case we're running Rails < 4.0 and this is a has_*
675
+ sub_obj)
676
+ # If still not found then create a new related object using this has_many collection
677
+ # (criteria.empty? ? nil : sub_next.new(criteria))
678
+ if sub_hm
679
+ sub_next = sub_hm
680
+ elsif assoc.macro == :has_one
681
+ # assoc.active_record.name.underscore is only there to support older Rails
682
+ # that doesn't do automatic inverse_of
683
+ ho_name = "#{assoc.inverse_of&.name || assoc.active_record.name.underscore}="
684
+ sub_next = assoc.klass.new(criteria)
685
+ to_be_saved << [sub_next, sub_next, ho_name, sub_obj]
686
+ elsif assoc.macro == :has_and_belongs_to_many ||
687
+ (assoc.macro == :has_many && assoc.options[:through])
688
+ # sub_next = sub_next.new(criteria)
689
+ # Search for one to wire up if it might already exist, otherwise create one
690
+ sub_next = assoc.klass.find_by(criteria) || assoc.klass.new(criteria)
691
+ to_be_saved << [sub_next, :has_and_belongs_to_many, assoc.name, sub_obj]
692
+ else
693
+ # Two other methods that are possible to check for here are :conditions and
694
+ # :sanitized_conditions, which do not exist in Rails 4.0 and later.
695
+ sub_next = if assoc.respond_to?(:require_association)
696
+ # With Rails < 4.0, sub_next could come in as an Array or a broken CollectionProxy
697
+ assoc.klass.new({ ::DutyFree::Extensions._fk_from(assoc) => sub_obj.send(sub_obj.class.primary_key) }.merge(criteria))
698
+ else
699
+ sub_next.proxy_association.reflection.instance_variable_set(:@foreign_key, ::DutyFree::Extensions._fk_from(assoc))
700
+ sub_next.new(criteria)
701
+ end
702
+ to_be_saved << [sub_next]
703
+ end
704
+ # else
705
+ # belongs_to for a found object, or a HMT
706
+ end
707
+ # # Incompatible with Rails < 4.2
708
+ # # Look for possible missing polymorphic detail
709
+ # # Maybe can test for this via assoc.through_reflection
710
+ # if assoc.is_a?(ActiveRecord::Reflection::ThroughReflection) &&
711
+ # (delegate = assoc.send(:delegate_reflection)&.active_record&.reflect_on_association(assoc.source_reflection_name)) &&
712
+ # delegate.options[:polymorphic]
713
+ # polymorphics << { parent: sub_next, child: sub_obj, type_col: delegate.foreign_type, id_col: delegate.foreign_key.to_s }
714
+ # end
715
+
716
+ # rubocop:disable Style/SoleNestedConditional
717
+ unless sub_next.nil?
718
+ # if sub_next.class.name == devise_class && # only for Devise users
719
+ # sub_next.email =~ Devise.email_regexp
720
+ # if existing.include?([sub_next.email])
721
+ # User already exists
722
+ # else
723
+ # sub_next.invite!
724
+ # end
725
+ # end
726
+ sub_objects[this_path] = sub_next if this_path.present?
727
+ end
728
+ # rubocop:enable Style/SoleNestedConditional
729
+ end
730
+ sub_obj = sub_next
731
+ end
732
+ next if sub_obj.nil?
711
733
 
712
- key << upn
713
- value << val
734
+ next unless sub_obj.respond_to?(sym = "#{v.name}=".to_sym)
735
+
736
+ col_type = sub_obj.class.columns_hash[v.name.to_s]&.type
737
+ if col_type.nil? && (virtual_columns = import_template[:virtual_columns]) &&
738
+ (virtual_columns = virtual_columns[this_path] || virtual_columns)
739
+ col_type = virtual_columns[v.name]
714
740
  end
715
- unless key.empty?
716
- s[key] = value
717
- utilised[key] = nil
741
+ if col_type == :boolean
742
+ if row[key].nil?
743
+ # Do nothing when it's nil
744
+ elsif %w[true t yes y].include?(row[key]&.strip&.downcase) # Used to cover 'on'
745
+ row[key] = true
746
+ elsif %w[false f no n].include?(row[key]&.strip&.downcase) # Used to cover 'off'
747
+ row[key] = false
748
+ else
749
+ row_errors[v.name] ||= []
750
+ row_errors[v.name] << "Boolean value \"#{row[key]}\" in column #{key + 1} not recognized"
751
+ end
718
752
  end
719
- else
720
- val = (unique_name = unique.to_s.titleize).start_with?(trim_prefix) &&
721
- cols.index(un = unique_name[trim_prefix.length..-1])
722
- if val
723
- s[[un]] = [val]
724
- utilised[[un]] = nil
753
+
754
+ is_yet_changed = sub_obj.changed?
755
+ sub_obj.send(sym, row[key])
756
+ # If this one is transitioning from having not changed to now having been changed,
757
+ # and is not a new one anyway that would already be lined up to get saved, then we
758
+ # mark it to now be saved.
759
+ to_be_saved << [sub_obj] if !sub_obj.new_record? && !is_yet_changed && sub_obj.changed?
760
+ # else
761
+ # puts " #{sub_obj.class.name} doesn't respond to #{sym}"
762
+ end
763
+ # Try to save final sub-object(s) if any exist
764
+ ::DutyFree::Extensions._save_pending(to_be_saved)
765
+
766
+ # Reinstate any missing polymorphic _type and _id values
767
+ polymorphics.each do |poly|
768
+ if !poly[:parent].new_record? || poly[:parent].save
769
+ poly[:child].send("#{poly[:type_col]}=".to_sym, poly[:parent].class.name)
770
+ poly[:child].send("#{poly[:id_col]}=".to_sym, poly[:parent].id)
725
771
  end
726
772
  end
727
- s
728
- end
729
- if defined_uniq.empty?
730
- (starred - utilised.keys).each { |star| defined_uniq[[star]] = [cols.index(star)] }
731
- # %%% puts "Tried to establish #{defined_uniq.inspect}"
773
+
774
+ # Give a window of opportunity to tweak user objects controlled by Devise
775
+ obj_class = obj.class
776
+ is_do_save = if obj_class.respond_to?(:before_devise_save) && obj_class.name == devise_class
777
+ obj_class.before_devise_save(obj, existing)
778
+ else
779
+ true
780
+ end
781
+
782
+ if obj.valid?
783
+ obj.save if is_do_save
784
+ # Snag back any changes to the unique columns. (For instance, Devise downcases email addresses.)
785
+ existing_unique = valid_unique.keys.inject([]) { |s, v| s << obj.send(v).to_s }
786
+ # Update the duplicate counts and inserted / updated results
787
+ counts[existing_unique] << row_num
788
+ (is_insert ? inserts : updates) << { row_num => existing_unique } if is_do_save
789
+ # Track this new object so we can properly sense any duplicates later
790
+ existing[existing_unique] = obj.id
791
+ else
792
+ row_errors.merge! obj.errors.messages
793
+ end
794
+ errors << { row_num => row_errors } unless row_errors.empty?
732
795
  end
733
- @defined_uniques[col_list] = defined_uniq
734
796
  end
735
- defined_uniq
797
+ duplicates = counts.each_with_object([]) do |v, s|
798
+ s += v.last[1..-1].map { |line_num| { line_num => v.first } } if v.last.count > 1
799
+ s
800
+ end
801
+ ret = { inserted: inserts, updated: updates, duplicates: duplicates, errors: errors }
802
+
803
+ # Check to see if they want to do anything after the import
804
+ # First if defined in the import_template, then if there is a method in the class,
805
+ # and finally (not yet implemented) a generic global after_import
806
+ my_after_import = import_template[:after_import]
807
+ my_after_import ||= respond_to?(:after_import) && method(:after_import)
808
+ # my_after_import ||= some generic my_after_import
809
+ if my_after_import
810
+ last_arg_idx = my_after_import.parameters.length - 1
811
+ arguments = [ret][0..last_arg_idx]
812
+ # rubocop:disable Lint/UselessAssignment
813
+ ret = ret2 if (ret2 = my_after_import.call(*arguments)).is_a?(Hash)
814
+ # rubocop:enable Lint/UselessAssignment
815
+ end
736
816
  end
737
- end # module ClassMethods
817
+ ret
818
+ end
819
+
820
+ def self._fk_from(assoc)
821
+ # Try first to trust whatever they've marked as being the foreign_key, and then look
822
+ # at the inverse's foreign key setting if available. In all cases don't accept
823
+ # anything that's not backed with a real column in the table.
824
+ col_names = assoc.klass.column_names
825
+ if (
826
+ (fk_name = assoc.options[:foreign_key]) ||
827
+ (fk_name = assoc.inverse_of&.options&.fetch(:foreign_key) { nil }) ||
828
+ (assoc.respond_to?(:foreign_key) && (fk_name = assoc.foreign_key)) ||
829
+ (fk_name = assoc.inverse_of&.foreign_key) ||
830
+ (fk_name = assoc.inverse_of&.association_foreign_key)
831
+ ) && col_names.include?(fk_name.to_s)
832
+ return fk_name
833
+ end
834
+
835
+ # Don't let this fool you -- we really are in search of the foreign key name here,
836
+ # and Rails 3.0 and older used some fairly interesting conventions, calling it instead
837
+ # the "primary_key_name"!
838
+ if assoc.respond_to?(:primary_key_name)
839
+ if (fk_name = assoc.primary_key_name) && col_names.include?(fk_name.to_s)
840
+ return fk_name
841
+ end
842
+ if (fk_name = assoc.inverse_of.primary_key_name) && col_names.include?(fk_name.to_s)
843
+ return fk_name
844
+ end
845
+ end
846
+
847
+ puts "* Wow, had no luck at all finding a foreign key for #{assoc.inspect}"
848
+ end
849
+
850
+ # unless build_tables.include?((foreign_class = tbs[2].class).underscore)
851
+ # build_table(build_tables, tbs[1].class, foreign_class, :has_one)
738
852
 
739
853
  # Called before building any object linked through a has_one or has_many so that foreign key IDs
740
854
  # can be added properly to those new objects. Finally at the end also called to save everything.
741
855
  def self._save_pending(to_be_saved)
742
856
  while (tbs = to_be_saved.pop)
743
857
  # puts "Will be calling #{tbs[1].class.name} #{tbs[1]&.id} .#{tbs[2]} #{tbs[3].class.name} #{tbs[3]&.id}"
744
-
745
- # Wire this one up if it had came from a has_one association
746
- tbs[1].send(tbs[2], tbs[3]) if tbs[0] == tbs[1]
858
+ # Wire this one up if it had come from a has_one
859
+ if tbs[0] == tbs[1]
860
+ tbs[1].class.has_one(tbs[2]) unless tbs[1].respond_to?(tbs[2])
861
+ tbs[1].send(tbs[2], tbs[3])
862
+ end
747
863
 
748
864
  ais = (tbs.first.class.respond_to?(:around_import_save) && tbs.first.class.method(:around_import_save)) ||
749
865
  (respond_to?(:around_import_save) && method(:around_import_save))
@@ -764,7 +880,17 @@ module DutyFree
764
880
  tbs[0] == tbs[1] || # From a has_one?
765
881
  tbs.first.new_record?
766
882
 
767
- tbs[1].send(tbs[2], tbs[3])
883
+ if tbs[1] == :has_and_belongs_to_many # also used by has_many :through associations
884
+ collection = tbs[3].send(tbs[2])
885
+ being_shoveled_id = tbs[0].send(tbs[0].class.primary_key)
886
+ if collection.empty? ||
887
+ !collection.pluck("#{(klass = collection.first.class).table_name}.#{klass.primary_key}").include?(being_shoveled_id)
888
+ collection << tbs[0]
889
+ # puts collection.inspect
890
+ end
891
+ else # Traditional belongs_to
892
+ tbs[1].send(tbs[2], tbs[3])
893
+ end
768
894
  end
769
895
  end
770
896
 
@@ -783,40 +909,71 @@ module DutyFree
783
909
  template_detail_columns
784
910
  end
785
911
 
786
- # Recurse and return two arrays -- one with all columns in sequence, and one a hierarchy of
787
- # nested hashes to be used with ActiveRecord's .joins() to facilitate export.
788
- def self._recurse_def(klass, array, import_template, order_by = [], assocs = [], joins = [], pre_prefix = '', prefix = '')
912
+ # Recurse and return three arrays -- one with all columns in sequence, and one a hierarchy of
913
+ # nested hashes to be used with ActiveRecord's .joins() to facilitate export, and finally
914
+ # one that lists tables that need to be built along the way.
915
+ def self._recurse_def(klass, cols, import_template, build_tables = nil, order_by = nil, assocs = [], joins = [], pre_prefix = '', prefix = '')
916
+ prefix = prefix[0..-2] if (is_build_table = prefix.end_with?('!'))
789
917
  prefixes = ::DutyFree::Util._prefix_join([pre_prefix, prefix])
790
- # Confirm we can actually navigate through this association
791
- prefix_assoc = (assocs.last&.klass || klass).reflect_on_association(prefix) if prefix.present?
792
- if prefix_assoc
793
- assocs = assocs.dup << prefix_assoc
794
- if prefix_assoc.macro == :has_many && (pk = prefix_assoc.active_record.primary_key)
795
- order_by << ["#{prefixes.tr('.', '_')}_", pk]
918
+
919
+ if prefix.present?
920
+ # An indication to build this table and model if it doesn't exist?
921
+ if is_build_table && build_tables
922
+ namespaces = prefix.split('::')
923
+ model_name = namespaces.map { |part| part.singularize.camelize }.join('::')
924
+ prefix = namespaces.last
925
+ # %%% If the model already exists, take belongs_to cues from it for building the table
926
+ if (is_need_model = Object.const_defined?(model_name)) &&
927
+ (is_need_table = ActiveRecord::ConnectionAdapters::SchemaStatements.table_exists?(
928
+ (path_name = ::DutyFree::Util._prefix_join([prefixes, prefix.pluralize]))
929
+ ))
930
+ is_build_table = false
931
+ else
932
+ build_tables[path_name] = [namespaces, is_need_model, is_need_table] unless build_tables.include?(path_name)
933
+ end
934
+ end
935
+ unless is_build_table && build_tables
936
+ # Confirm we can actually navigate through this association
937
+ prefix_assoc = (assocs.last&.klass || klass).reflect_on_association(prefix) if prefix.present?
938
+ if prefix_assoc
939
+ assocs = assocs.dup << prefix_assoc
940
+ if order_by && [:has_many, :has_and_belongs_to_many].include?(prefix_assoc.macro) &&
941
+ (pk = prefix_assoc.active_record.primary_key)
942
+ order_by << ["#{prefixes.tr('.', '_')}_", pk]
943
+ end
944
+ end
796
945
  end
797
946
  end
798
- array = array.inject([]) do |s, col|
947
+ cols = cols.inject([]) do |s, col|
799
948
  s + if col.is_a?(Hash)
800
949
  col.inject([]) do |s2, v|
801
- joins << { v.first.to_sym => (joins_array = []) }
802
- s2 + _recurse_def(klass, (v.last.is_a?(Array) ? v.last : [v.last]), import_template, order_by, assocs, joins_array, prefixes, v.first.to_sym).first
950
+ if order_by
951
+ # Find what the type is for this guy
952
+ next_klass = (assocs.last&.klass || klass).reflect_on_association(v.first)&.klass
953
+ # Used to be: { v.first.to_sym => (joins_array = []) }
954
+ joins << { v.first.to_sym => (joins_array = [next_klass]) }
955
+ end
956
+ s2 + _recurse_def(klass, (v.last.is_a?(Array) ? v.last : [v.last]), import_template, build_tables, order_by, assocs, joins_array, prefixes, v.first.to_sym).first
803
957
  end
804
958
  elsif col.nil?
805
959
  if assocs.empty?
806
960
  []
807
961
  else
808
962
  # Bring in from another class
809
- joins << { prefix => (joins_array = []) }
963
+ # Used to be: { prefix => (joins_array = []) }
964
+ # %%% Probably need a next_klass thing like above
965
+ joins << { prefix => (joins_array = [klass]) } if order_by
810
966
  # %%% Also bring in uniques and requireds
811
- _recurse_def(klass, assocs.last.klass::IMPORT_TEMPLATE[:all], import_template, order_by, assocs, joins_array, prefixes).first
967
+ _recurse_def(klass, assocs.last.klass::IMPORT_TEMPLATE[:all], import_template, build_tables, order_by, assocs, joins_array, prefixes).first
812
968
  end
813
969
  else
814
970
  [::DutyFree::Column.new(col, pre_prefix, prefix, assocs, klass, import_template[:as])]
815
971
  end
816
972
  end
817
- [array, joins]
973
+ [cols, joins]
818
974
  end
819
975
  end # module Extensions
976
+ # rubocop:enable Style/CommentedKeyword
820
977
 
821
978
  # Rails < 4.0 doesn't have ActiveRecord::RecordNotUnique, so use the more generic ActiveRecord::ActiveRecordError instead
822
979
  ar_not_unique_error = ActiveRecord.const_defined?('RecordNotUnique') ? ActiveRecord::RecordNotUnique : ActiveRecord::ActiveRecordError