duty_free 1.0.7 → 1.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/duty_free/column.rb +2 -2
- data/lib/duty_free/extensions.rb +603 -446
- data/lib/duty_free/suggest_template.rb +381 -170
- data/lib/duty_free/util.rb +129 -11
- data/lib/duty_free/version_number.rb +1 -1
- data/lib/duty_free.rb +227 -52
- data/lib/generators/duty_free/model_generator.rb +349 -0
- data/lib/generators/duty_free/templates/create_versions.rb.erb +2 -2
- metadata +10 -29
data/lib/duty_free/extensions.rb
CHANGED
@@ -6,7 +6,11 @@ require 'duty_free/suggest_template'
|
|
6
6
|
|
7
7
|
# :nodoc:
|
8
8
|
module DutyFree
|
9
|
+
# rubocop:disable Style/CommentedKeyword
|
9
10
|
module Extensions
|
11
|
+
MAX_ID = Arel.sql('MAX(id)')
|
12
|
+
IS_AMOEBA = Gem.loaded_specs['amoeba']
|
13
|
+
|
10
14
|
def self.included(base)
|
11
15
|
base.send :extend, ClassMethods
|
12
16
|
base.send :extend, ::DutyFree::SuggestTemplate::ClassMethods
|
@@ -14,10 +18,6 @@ module DutyFree
|
|
14
18
|
|
15
19
|
# :nodoc:
|
16
20
|
module ClassMethods
|
17
|
-
MAX_ID = Arel.sql('MAX(id)')
|
18
|
-
# def self.extended(model)
|
19
|
-
# end
|
20
|
-
|
21
21
|
# Export at least column header, and optionally include all existing data as well
|
22
22
|
def df_export(is_with_data = true, import_template = nil, use_inner_joins = false)
|
23
23
|
use_inner_joins = true unless respond_to?(:left_joins)
|
@@ -53,8 +53,12 @@ module DutyFree
|
|
53
53
|
if is_with_data
|
54
54
|
order_by = []
|
55
55
|
order_by << ['_', primary_key] if primary_key
|
56
|
+
all = import_template[:all] || import_template[:all!]
|
56
57
|
# Automatically create a JOINs strategy and select list to get back all related rows
|
57
|
-
template_cols, template_joins = ::DutyFree::Extensions._recurse_def(self,
|
58
|
+
template_cols, template_joins = ::DutyFree::Extensions._recurse_def(self, all, import_template, nil, order_by)
|
59
|
+
# We do this so early here because it removes type objects from template_joins so then
|
60
|
+
# template_joins can immediately be used for inner and outer JOINs.
|
61
|
+
our_names = [[self, '_']] + ::DutyFree::Util._recurse_arel(template_joins)
|
58
62
|
relation = use_inner_joins ? joins(template_joins) : left_joins(template_joins)
|
59
63
|
|
60
64
|
# So we can properly create the SELECT list, create a mapping between our
|
@@ -71,12 +75,27 @@ module DutyFree
|
|
71
75
|
# With AR < 3.2, "froms" brings up the top node, an Arel::Nodes::InnerJoin
|
72
76
|
::DutyFree::Util._recurse_arel(core.froms)
|
73
77
|
end
|
74
|
-
our_names
|
75
|
-
|
78
|
+
# Make sure our_names lines up with the arel_alias_name by comparing the ActiveRecord type.
|
79
|
+
# AR < 5.0 behaves differently than newer versions, and AR 5.0 and 5.1 have a bug in the
|
80
|
+
# way the types get determined, so if we want perfect results then we must compensate for
|
81
|
+
# these foibles. Thank goodness that AR 5.2 and later find the proper type and make it
|
82
|
+
# available through the type_caster object, which is what we use when building the list of
|
83
|
+
# arel_alias_names.
|
84
|
+
mapping = arel_alias_names.each_with_object({}) do |arel_alias_name, s|
|
85
|
+
if our_names.first&.first == arel_alias_name.first
|
86
|
+
s[our_names.first.last] = arel_alias_name.last
|
87
|
+
our_names.shift
|
88
|
+
end
|
89
|
+
s
|
90
|
+
end
|
76
91
|
relation = (order_by.empty? ? relation : relation.order(order_by.map { |o| "#{mapping[o.first]}.#{o.last}" }))
|
77
92
|
# puts mapping.inspect
|
78
93
|
# puts relation.dup.select(template_cols.map { |x| x.to_s(mapping) }).to_sql
|
79
|
-
|
94
|
+
|
95
|
+
# Allow customisation of query before running it
|
96
|
+
relation = yield(relation, mapping) if block_given?
|
97
|
+
|
98
|
+
relation&.select(template_cols.map { |x| x.to_s(mapping) })&.each do |result|
|
80
99
|
rows << ::DutyFree::Extensions._template_columns(self, import_template).map do |col|
|
81
100
|
value = result.send(col)
|
82
101
|
case value
|
@@ -93,399 +112,57 @@ module DutyFree
|
|
93
112
|
rows
|
94
113
|
end
|
95
114
|
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
instance_variable_set(:@valid_uniques, nil)
|
100
|
-
|
101
|
-
import_template ||= if constants.include?(:IMPORT_TEMPLATE)
|
102
|
-
self::IMPORT_TEMPLATE
|
103
|
-
else
|
104
|
-
suggest_template(0, false, false)
|
105
|
-
end
|
106
|
-
# puts "Chose #{import_template}"
|
107
|
-
inserts = []
|
108
|
-
updates = []
|
109
|
-
counts = Hash.new { |h, k| h[k] = [] }
|
110
|
-
errors = []
|
111
|
-
|
112
|
-
is_first = true
|
113
|
-
uniques = nil
|
114
|
-
cols = nil
|
115
|
-
starred = []
|
116
|
-
partials = []
|
117
|
-
all = import_template[:all]
|
118
|
-
keepers = {}
|
119
|
-
valid_unique = nil
|
120
|
-
existing = {}
|
121
|
-
devise_class = ''
|
122
|
-
ret = nil
|
123
|
-
|
124
|
-
# Multi-tenancy gem Apartment can be used if there are separate schemas per tenant
|
125
|
-
reference_models = if Object.const_defined?('Apartment')
|
126
|
-
Apartment.excluded_models
|
127
|
-
else
|
128
|
-
[]
|
129
|
-
end
|
130
|
-
|
131
|
-
if Object.const_defined?('Devise')
|
132
|
-
Object.const_get('Devise') # If this fails, devise_class will remain a blank string.
|
133
|
-
devise_class = Devise.mappings.values.first.class_name
|
134
|
-
reference_models -= [devise_class]
|
135
|
-
else
|
136
|
-
devise_class = ''
|
137
|
-
end
|
138
|
-
|
139
|
-
# Did they give us a filename?
|
140
|
-
if data.is_a?(String)
|
141
|
-
# Filenames with full paths can not be longer than 4096 characters, and can not
|
142
|
-
# include newline characters
|
143
|
-
data = if data.length <= 4096 && !data.index('\n')
|
144
|
-
File.open(data)
|
145
|
-
else
|
146
|
-
# Any multi-line string is likely CSV data
|
147
|
-
# %%% Test to see if TAB characters are present on the first line, instead of commas
|
148
|
-
CSV.new(data)
|
149
|
-
end
|
150
|
-
end
|
151
|
-
# Or perhaps us a file?
|
152
|
-
if data.is_a?(File)
|
153
|
-
# Use the "roo" gem if it's available
|
154
|
-
data = if Object.const_defined?('Roo::Spreadsheet', { csv_options: { encoding: 'bom|utf-8' } })
|
155
|
-
Roo::Spreadsheet.open(data)
|
156
|
-
else
|
157
|
-
# Otherwise generic CSV parsing
|
158
|
-
require 'csv' unless Object.const_defined?('CSV')
|
159
|
-
CSV.open(data)
|
160
|
-
end
|
161
|
-
end
|
162
|
-
|
163
|
-
# Will show as just one transaction when using auditing solutions such as PaperTrail
|
164
|
-
ActiveRecord::Base.transaction do
|
165
|
-
# Check to see if they want to do anything before the whole import
|
166
|
-
# First if defined in the import_template, then if there is a method in the class,
|
167
|
-
# and finally (not yet implemented) a generic global before_import
|
168
|
-
my_before_import = import_template[:before_import]
|
169
|
-
my_before_import ||= respond_to?(:before_import) && method(:before_import)
|
170
|
-
# my_before_import ||= some generic my_before_import
|
171
|
-
if my_before_import
|
172
|
-
last_arg_idx = my_before_import.parameters.length - 1
|
173
|
-
arguments = [data, import_template][0..last_arg_idx]
|
174
|
-
data = ret if (ret = my_before_import.call(*arguments)).is_a?(Enumerable)
|
175
|
-
end
|
176
|
-
data.each_with_index do |row, row_num|
|
177
|
-
row_errors = {}
|
178
|
-
if is_first # Anticipate that first row has column names
|
179
|
-
uniques = import_template[:uniques]
|
180
|
-
|
181
|
-
# Look for UTF-8 BOM in very first cell
|
182
|
-
row[0] = row[0][3..-1] if row[0].start_with?([239, 187, 191].pack('U*'))
|
183
|
-
# How about a first character of FEFF or FFFE to support UTF-16 BOMs?
|
184
|
-
# FE FF big-endian (standard)
|
185
|
-
# FF FE little-endian
|
186
|
-
row[0] = row[0][1..-1] if [65_279, 65_534].include?(row[0][0].ord)
|
187
|
-
cols = row.map { |col| (col || '').strip }
|
188
|
-
|
189
|
-
# Unique column combinations can be called out explicitly in models using uniques: {...}, or to just
|
190
|
-
# define one column at a time simply mark with an asterisk.
|
191
|
-
# Track and clean up stars
|
192
|
-
starred = cols.select do |col|
|
193
|
-
if col[0] == '*'
|
194
|
-
col.slice!(0)
|
195
|
-
col.strip!
|
196
|
-
end
|
197
|
-
end
|
198
|
-
partials = cols.select do |col|
|
199
|
-
if col[0] == '~'
|
200
|
-
col.slice!(0)
|
201
|
-
col.strip!
|
202
|
-
end
|
203
|
-
end
|
204
|
-
cols.map! { |col| ::DutyFree::Util._clean_name(col, import_template[:as]) }
|
205
|
-
defined_uniques(uniques, cols, cols.join('|'), starred)
|
206
|
-
# Make sure that at least half of them match what we know as being good column names
|
207
|
-
template_column_objects = ::DutyFree::Extensions._recurse_def(self, import_template[:all], import_template).first
|
208
|
-
cols.each_with_index do |col, idx|
|
209
|
-
# prefixes = col_detail.pre_prefix + (col_detail.prefix.blank? ? [] : [col_detail.prefix])
|
210
|
-
keepers[idx] = template_column_objects.find { |col_obj| col_obj.titleize == col }
|
211
|
-
# puts "Could not find a match for column #{idx + 1}, #{col}" if keepers[idx].nil?
|
212
|
-
end
|
213
|
-
raise ::DutyFree::LessThanHalfAreMatchingColumnsError, I18n.t('import.altered_import_template_coumns') if keepers.length < (cols.length / 2) - 1
|
214
|
-
|
215
|
-
# Returns just the first valid unique lookup set if there are multiple
|
216
|
-
valid_unique = find_existing(uniques, cols, starred, import_template, keepers, false)
|
217
|
-
# Make a lookup from unique values to specific IDs
|
218
|
-
existing = pluck(*([:id] + valid_unique.keys)).each_with_object(existing) do |v, s|
|
219
|
-
s[v[1..-1].map(&:to_s)] = v.first
|
220
|
-
s
|
221
|
-
end
|
222
|
-
is_first = false
|
223
|
-
else # Normal row of data
|
224
|
-
is_insert = false
|
225
|
-
existing_unique = valid_unique.inject([]) do |s, v|
|
226
|
-
s << if v.last.is_a?(Array)
|
227
|
-
v.last[0].where(v.last[1] => row[v.last[2]]).limit(1).pluck(MAX_ID).first.to_s
|
228
|
-
else
|
229
|
-
row[v.last].to_s
|
230
|
-
end
|
231
|
-
end
|
232
|
-
to_be_saved = []
|
233
|
-
# Check to see if they want to preprocess anything
|
234
|
-
existing_unique = @before_process.call(valid_unique, existing_unique) if @before_process ||= import_template[:before_process]
|
235
|
-
if (criteria = existing[existing_unique])
|
236
|
-
obj = find(criteria)
|
237
|
-
else
|
238
|
-
is_insert = true
|
239
|
-
to_be_saved << [obj = new]
|
240
|
-
end
|
241
|
-
sub_obj = nil
|
242
|
-
is_has_one = false
|
243
|
-
has_ones = []
|
244
|
-
polymorphics = []
|
245
|
-
sub_objects = {}
|
246
|
-
this_path = nil
|
247
|
-
keepers.each do |key, v|
|
248
|
-
next if v.nil?
|
249
|
-
|
250
|
-
sub_obj = obj
|
251
|
-
this_path = +''
|
252
|
-
# puts "p: #{v.path}"
|
253
|
-
v.path.each_with_index do |path_part, idx|
|
254
|
-
this_path << (this_path.blank? ? path_part.to_s : ",#{path_part}")
|
255
|
-
unless (sub_next = sub_objects[this_path])
|
256
|
-
# Check if we're hitting reference data / a lookup thing
|
257
|
-
assoc = v.prefix_assocs[idx]
|
258
|
-
# belongs_to some lookup (reference) data
|
259
|
-
if assoc && reference_models.include?(assoc.class_name)
|
260
|
-
lookup_match = assoc.klass.find_by(v.name => row[key])
|
261
|
-
# Do a partial match if this column allows for it
|
262
|
-
# and we only find one matching result.
|
263
|
-
if lookup_match.nil? && partials.include?(v.titleize)
|
264
|
-
lookup_match ||= assoc.klass.where("#{v.name} LIKE '#{row[key]}%'")
|
265
|
-
lookup_match = (lookup_match.length == 1 ? lookup_match.first : nil)
|
266
|
-
end
|
267
|
-
sub_obj.send("#{path_part}=", lookup_match) unless lookup_match.nil?
|
268
|
-
# Reference data from the public level means we stop here
|
269
|
-
sub_obj = nil
|
270
|
-
break
|
271
|
-
end
|
272
|
-
# Get existing related object, or create a new one
|
273
|
-
# This first part works for belongs_to. has_many and has_one get sorted below.
|
274
|
-
if (sub_next = sub_obj.send(path_part)).nil? && assoc.belongs_to?
|
275
|
-
klass = Object.const_get(assoc&.class_name)
|
276
|
-
# Try to find a unique item if one is referenced
|
277
|
-
sub_next = nil
|
278
|
-
begin
|
279
|
-
trim_prefix = v.titleize[0..-(v.name.length + 2)]
|
280
|
-
trim_prefix << ' ' unless trim_prefix.blank?
|
281
|
-
sub_next, criteria = klass.find_existing(uniques, cols, starred, import_template, keepers, nil, row, klass, all, trim_prefix)
|
282
|
-
rescue ::DutyFree::NoUniqueColumnError
|
283
|
-
end
|
284
|
-
# puts "#{v.path} #{criteria.inspect}"
|
285
|
-
bt_name = "#{path_part}="
|
286
|
-
unless sub_next || (klass == sub_obj.class && criteria.empty?)
|
287
|
-
sub_next = klass.new(criteria || {})
|
288
|
-
to_be_saved << [sub_next, sub_obj, bt_name, sub_next]
|
289
|
-
end
|
290
|
-
# This wires it up in memory, but doesn't yet put the proper foreign key ID in
|
291
|
-
# place when the primary object is a new one (and thus not having yet been saved
|
292
|
-
# doesn't yet have an ID).
|
293
|
-
# binding.pry if !sub_next.new_record? && sub_next.name == 'Squidget Widget' # !sub_obj.changed?
|
294
|
-
# # %%% Question is -- does the above set changed? when a foreign key is not yet set
|
295
|
-
# # and only the in-memory object has changed?
|
296
|
-
is_yet_changed = sub_obj.changed?
|
297
|
-
sub_obj.send(bt_name, sub_next)
|
298
|
-
|
299
|
-
# We need this in the case of updating the primary object across a belongs_to when
|
300
|
-
# the foreign one already exists and is not otherwise changing, such as when it is
|
301
|
-
# not a new one, so wouldn't otherwise be getting saved.
|
302
|
-
to_be_saved << [sub_obj] if !sub_obj.new_record? && !is_yet_changed && sub_obj.changed?
|
303
|
-
# From a has_many or has_one?
|
304
|
-
# Rails 4.0 and later can do: sub_next.is_a?(ActiveRecord::Associations::CollectionProxy)
|
305
|
-
elsif [:has_many, :has_one].include?(assoc.macro) && !assoc.options[:through]
|
306
|
-
::DutyFree::Extensions._save_pending(to_be_saved)
|
307
|
-
# Try to find a unique item if one is referenced
|
308
|
-
# %%% There is possibility that when bringing in related classes using a nil
|
309
|
-
# in IMPORT_TEMPLATE[:all] that this will break. Need to test deeply nested things.
|
310
|
-
start = (v.pre_prefix.blank? ? 0 : v.pre_prefix.length)
|
311
|
-
trim_prefix = v.titleize[start..-(v.name.length + 2)]
|
312
|
-
trim_prefix << ' ' unless trim_prefix.blank?
|
313
|
-
# assoc.inverse_of is the belongs_to side of the has_many train we came in here on.
|
314
|
-
sub_hm, criteria = assoc.klass.find_existing(uniques, cols, starred, import_template, keepers, assoc.inverse_of, row, sub_next, all, trim_prefix)
|
315
|
-
# If still not found then create a new related object using this has_many collection
|
316
|
-
# (criteria.empty? ? nil : sub_next.new(criteria))
|
317
|
-
if sub_hm
|
318
|
-
sub_next = sub_hm
|
319
|
-
elsif assoc.macro == :has_one
|
320
|
-
# assoc.active_record.name.underscore is only there to support older Rails
|
321
|
-
# that doesn't do automatic inverse_of
|
322
|
-
ho_name = "#{assoc.inverse_of&.name || assoc.active_record.name.underscore}="
|
323
|
-
sub_next = assoc.klass.new(criteria)
|
324
|
-
to_be_saved << [sub_next, sub_next, ho_name, sub_obj]
|
325
|
-
else
|
326
|
-
# Two other methods that are possible to check for here are :conditions and
|
327
|
-
# :sanitized_conditions, which do not exist in Rails 4.0 and later.
|
328
|
-
sub_next = if assoc.respond_to?(:require_association)
|
329
|
-
# With Rails < 4.0, sub_next could come in as an Array or a broken CollectionProxy
|
330
|
-
assoc.klass.new({ fk_from(assoc) => sub_obj.send(sub_obj.class.primary_key) }.merge(criteria))
|
331
|
-
else
|
332
|
-
sub_next.proxy_association.reflection.instance_variable_set(:@foreign_key, fk_from(assoc))
|
333
|
-
sub_next.new(criteria)
|
334
|
-
end
|
335
|
-
to_be_saved << [sub_next]
|
336
|
-
end
|
337
|
-
end
|
338
|
-
# Look for possible missing polymorphic detail
|
339
|
-
# Maybe can test for this via assoc.through_reflection
|
340
|
-
if assoc.is_a?(ActiveRecord::Reflection::ThroughReflection) &&
|
341
|
-
(delegate = assoc.send(:delegate_reflection)&.active_record&.reflect_on_association(assoc.source_reflection_name)) &&
|
342
|
-
delegate.options[:polymorphic]
|
343
|
-
polymorphics << { parent: sub_next, child: sub_obj, type_col: delegate.foreign_type, id_col: delegate.foreign_key.to_s }
|
344
|
-
end
|
345
|
-
unless sub_next.nil?
|
346
|
-
# if sub_next.class.name == devise_class && # only for Devise users
|
347
|
-
# sub_next.email =~ Devise.email_regexp
|
348
|
-
# if existing.include?([sub_next.email])
|
349
|
-
# User already exists
|
350
|
-
# else
|
351
|
-
# sub_next.invite!
|
352
|
-
# end
|
353
|
-
# end
|
354
|
-
sub_objects[this_path] = sub_next if this_path.present?
|
355
|
-
end
|
356
|
-
end
|
357
|
-
sub_obj = sub_next
|
358
|
-
end
|
359
|
-
next if sub_obj.nil?
|
115
|
+
def df_import(data, import_template = nil, insert_only = false)
|
116
|
+
::DutyFree::Extensions.import(self, data, import_template, insert_only)
|
117
|
+
end
|
360
118
|
|
361
|
-
|
119
|
+
private
|
362
120
|
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
else
|
376
|
-
row_errors[v.name] ||= []
|
377
|
-
row_errors[v.name] << "Boolean value \"#{row[key]}\" in column #{key + 1} not recognized"
|
378
|
-
end
|
379
|
-
end
|
121
|
+
def _defined_uniques(uniques, cols = [], col_list = nil, starred = [], trim_prefix = '')
|
122
|
+
col_list ||= cols.join('|')
|
123
|
+
unless (defined_uniq = (@defined_uniques ||= {})[col_list])
|
124
|
+
utilised = {} # Track columns that have been referenced thusfar
|
125
|
+
defined_uniq = uniques.each_with_object({}) do |unique, s|
|
126
|
+
if unique.is_a?(Array)
|
127
|
+
key = []
|
128
|
+
value = []
|
129
|
+
unique.each do |unique_part|
|
130
|
+
val = (unique_part_name = unique_part.to_s.titleize).start_with?(trim_prefix) &&
|
131
|
+
cols.index(upn = unique_part_name[trim_prefix.length..-1])
|
132
|
+
next unless val
|
380
133
|
|
381
|
-
|
382
|
-
|
383
|
-
# If this one is transitioning from having not changed to now having been changed,
|
384
|
-
# and is not a new one anyway that would already be lined up to get saved, then we
|
385
|
-
# mark it to now be saved.
|
386
|
-
to_be_saved << [sub_obj] if !sub_obj.new_record? && !is_yet_changed && sub_obj.changed?
|
387
|
-
# else
|
388
|
-
# puts " #{sub_obj.class.name} doesn't respond to #{sym}"
|
134
|
+
key << upn
|
135
|
+
value << val
|
389
136
|
end
|
390
|
-
|
391
|
-
|
392
|
-
|
393
|
-
# Reinstate any missing polymorphic _type and _id values
|
394
|
-
polymorphics.each do |poly|
|
395
|
-
if !poly[:parent].new_record? || poly[:parent].save
|
396
|
-
poly[:child].send("#{poly[:type_col]}=".to_sym, poly[:parent].class.name)
|
397
|
-
poly[:child].send("#{poly[:id_col]}=".to_sym, poly[:parent].id)
|
398
|
-
end
|
137
|
+
unless key.empty?
|
138
|
+
s[key] = value
|
139
|
+
utilised[key] = nil
|
399
140
|
end
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
true
|
407
|
-
end
|
408
|
-
|
409
|
-
if obj.valid?
|
410
|
-
obj.save if is_do_save
|
411
|
-
# Snag back any changes to the unique columns. (For instance, Devise downcases email addresses.)
|
412
|
-
existing_unique = valid_unique.keys.inject([]) { |s, v| s << obj.send(v).to_s }
|
413
|
-
# Update the duplicate counts and inserted / updated results
|
414
|
-
counts[existing_unique] << row_num
|
415
|
-
(is_insert ? inserts : updates) << { row_num => existing_unique } if is_do_save
|
416
|
-
# Track this new object so we can properly sense any duplicates later
|
417
|
-
existing[existing_unique] = obj.id
|
418
|
-
else
|
419
|
-
row_errors.merge! obj.errors.messages
|
141
|
+
else
|
142
|
+
val = (unique_name = unique.to_s.titleize).start_with?(trim_prefix) &&
|
143
|
+
cols.index(un = unique_name[trim_prefix.length..-1])
|
144
|
+
if val
|
145
|
+
s[[un]] = [val]
|
146
|
+
utilised[[un]] = nil
|
420
147
|
end
|
421
|
-
errors << { row_num => row_errors } unless row_errors.empty?
|
422
148
|
end
|
423
|
-
end
|
424
|
-
duplicates = counts.each_with_object([]) do |v, s|
|
425
|
-
s += v.last[1..-1].map { |line_num| { line_num => v.first } } if v.last.count > 1
|
426
149
|
s
|
427
150
|
end
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
# First if defined in the import_template, then if there is a method in the class,
|
432
|
-
# and finally (not yet implemented) a generic global after_import
|
433
|
-
my_after_import = import_template[:after_import]
|
434
|
-
my_after_import ||= respond_to?(:after_import) && method(:after_import)
|
435
|
-
# my_after_import ||= some generic my_after_import
|
436
|
-
if my_after_import
|
437
|
-
last_arg_idx = my_after_import.parameters.length - 1
|
438
|
-
arguments = [ret][0..last_arg_idx]
|
439
|
-
ret = ret2 if (ret2 = my_after_import.call(*arguments)).is_a?(Hash)
|
440
|
-
end
|
441
|
-
end
|
442
|
-
ret
|
443
|
-
end
|
444
|
-
|
445
|
-
def fk_from(assoc)
|
446
|
-
# Try first to trust whatever they've marked as being the foreign_key, and then look
|
447
|
-
# at the inverse's foreign key setting if available. In all cases don't accept
|
448
|
-
# anything that's not backed with a real column in the table.
|
449
|
-
col_names = assoc.klass.column_names
|
450
|
-
if (fk_name = assoc.options[:foreign_key]) && col_names.include?(fk_name.to_s)
|
451
|
-
return fk_name
|
452
|
-
end
|
453
|
-
if (fk_name = assoc.inverse_of&.options&.fetch(:foreign_key) { nil }) &&
|
454
|
-
col_names.include?(fk_name.to_s)
|
455
|
-
return fk_name
|
456
|
-
end
|
457
|
-
if assoc.respond_to?(:foreign_key) && (fk_name = assoc.foreign_key) &&
|
458
|
-
col_names.include?(fk_name.to_s)
|
459
|
-
return fk_name
|
460
|
-
end
|
461
|
-
if (fk_name = assoc.inverse_of&.foreign_key) &&
|
462
|
-
col_names.include?(fk_name.to_s)
|
463
|
-
return fk_name
|
464
|
-
end
|
465
|
-
if (fk_name = assoc.inverse_of&.association_foreign_key) &&
|
466
|
-
col_names.include?(fk_name.to_s)
|
467
|
-
return fk_name
|
468
|
-
end
|
469
|
-
# Don't let this fool you -- we really are in search of the foreign key name here,
|
470
|
-
# and Rails 3.0 and older used some fairly interesting conventions, calling it instead
|
471
|
-
# the "primary_key_name"!
|
472
|
-
if assoc.respond_to?(:primary_key_name)
|
473
|
-
if (fk_name = assoc.primary_key_name) && col_names.include?(fk_name.to_s)
|
474
|
-
return fk_name
|
475
|
-
end
|
476
|
-
if (fk_name = assoc.inverse_of.primary_key_name) && col_names.include?(fk_name.to_s)
|
477
|
-
return fk_name
|
151
|
+
if defined_uniq.empty?
|
152
|
+
(starred - utilised.keys).each { |star| defined_uniq[[star]] = [cols.index(star)] }
|
153
|
+
# %%% puts "Tried to establish #{defined_uniq.inspect}"
|
478
154
|
end
|
155
|
+
@defined_uniques[col_list] = defined_uniq
|
479
156
|
end
|
480
|
-
|
481
|
-
puts "* Wow, had no luck at all finding a foreign key for #{assoc.inspect}"
|
157
|
+
defined_uniq
|
482
158
|
end
|
483
159
|
|
484
160
|
# For use with importing, based on the provided column list calculate all valid combinations
|
485
161
|
# of unique columns. If there is no valid combination, throws an error.
|
486
|
-
# Returns an object found by this means.
|
487
|
-
def
|
488
|
-
|
162
|
+
# Returns an object found by this means, as well as the criteria that was used to find it.
|
163
|
+
def _find_existing(uniques, cols, starred, import_template, keepers, train_we_came_in_here_on, insert_only,
|
164
|
+
row = nil, klass_or_collection = nil, template_all = nil, trim_prefix = '',
|
165
|
+
assoc = nil, base_obj = nil)
|
489
166
|
unless trim_prefix.blank?
|
490
167
|
cols = cols.map { |c| c.start_with?(trim_prefix) ? c[trim_prefix.length..-1] : nil }
|
491
168
|
starred = starred.each_with_object([]) do |v, s|
|
@@ -536,7 +213,7 @@ module DutyFree
|
|
536
213
|
# folks based on having their email as a unique identifier, and other folks by having a
|
537
214
|
# combination of their name and street address as unique, and use both of those possible
|
538
215
|
# unique variations to update phone numbers, and do that all as a part of one import.)
|
539
|
-
all_vus =
|
216
|
+
all_vus = _defined_uniques(uniques, cols, col_list, starred, trim_prefix)
|
540
217
|
|
541
218
|
# %%% Ultimately may consider making this recursive
|
542
219
|
reflect_on_all_associations.each do |sn_bt|
|
@@ -586,7 +263,8 @@ module DutyFree
|
|
586
263
|
# If we're processing a row then this list of foreign key column name entries, named such as
|
587
264
|
# "order_id" or "product_id" instead of column-specific stuff like "Order Date" and "Product Name",
|
588
265
|
# is kept until the last and then gets merged on top of the other criteria before being returned.
|
589
|
-
|
266
|
+
fk_name = sn_bt.foreign_key
|
267
|
+
bt_criteria[fk_name] = fk_id unless bt_criteria.include?(fk_name)
|
590
268
|
|
591
269
|
# Check to see if belongs_tos are generally required on this specific table
|
592
270
|
bt_req_by_default = sn_bt.klass.respond_to?(:belongs_to_required_by_default) &&
|
@@ -606,7 +284,7 @@ module DutyFree
|
|
606
284
|
(sn_bt.options[:optional] || !bt_req_by_default)
|
607
285
|
|
608
286
|
# Add to the criteria
|
609
|
-
criteria[fk_name] = fk_id
|
287
|
+
criteria[fk_name] = fk_id if insert_only && !criteria.include?(fk_name)
|
610
288
|
end
|
611
289
|
end
|
612
290
|
|
@@ -656,19 +334,63 @@ module DutyFree
|
|
656
334
|
# Find by all corresponding columns
|
657
335
|
if (row_value = row[v])
|
658
336
|
new_criteria_all_nil = false
|
659
|
-
criteria[k_sym] = row_value # The data
|
337
|
+
criteria[k_sym] = row_value # The data
|
660
338
|
end
|
661
339
|
end
|
662
340
|
end
|
341
|
+
# puts uniq_lookups.inspect
|
663
342
|
|
664
|
-
return uniq_lookups.merge(criteria) if only_valid_uniques
|
665
|
-
|
343
|
+
return [uniq_lookups.merge(criteria), bt_criteria] if only_valid_uniques
|
666
344
|
# If there's nothing to match upon then we're out
|
667
|
-
return [nil, {}] if new_criteria_all_nil
|
345
|
+
return [nil, {}, {}] if new_criteria_all_nil
|
668
346
|
|
669
347
|
# With this criteria, find any matching has_many row we can so we can update it.
|
670
348
|
# First try directly looking it up through ActiveRecord.
|
671
|
-
|
349
|
+
klass_or_collection ||= self # Comes in as nil for has_one with no object yet attached
|
350
|
+
# HABTM proxy
|
351
|
+
# binding.pry if klass_or_collection.is_a?(ActiveRecord::Associations::CollectionProxy)
|
352
|
+
# if klass_or_collection.respond_to?(:proxy_association) && klass_or_collection.proxy_association.options.include?(:through)
|
353
|
+
# klass_or_collection.proxy_association.association_scope.to_a
|
354
|
+
|
355
|
+
# if assoc.respond_to?(:require_association) && klass_or_collection.is_a?(Array)
|
356
|
+
# # else
|
357
|
+
# # klass_or_collection = assoc.klass
|
358
|
+
# end
|
359
|
+
# end
|
360
|
+
found_object = case klass_or_collection
|
361
|
+
when ActiveRecord::Base # object from a has_one?
|
362
|
+
existing_object = klass_or_collection
|
363
|
+
klass_or_collection = klass_or_collection.class
|
364
|
+
other_object = klass_or_collection.find_by(criteria)
|
365
|
+
pk = klass_or_collection.primary_key
|
366
|
+
existing_object.send(pk) == other_object&.send(pk) ? existing_object : other_object
|
367
|
+
when Array # has_* in AR < 4.0
|
368
|
+
# Old AR doesn't have a CollectionProxy that can do any of this on its own.
|
369
|
+
base_id = base_obj.send(base_obj.class.primary_key)
|
370
|
+
if assoc.macro == :has_and_belongs_to_many || (assoc.macro == :has_many && assoc.options[:through])
|
371
|
+
# Find all association foreign keys, then find or create the foreign object
|
372
|
+
# based on criteria, and finally put an entry with both foreign keys into
|
373
|
+
# the associative table unless it already exists.
|
374
|
+
ajt = assoc.through_reflection&.table_name || assoc.join_table
|
375
|
+
fk = assoc.foreign_key
|
376
|
+
afk = assoc.association_foreign_key
|
377
|
+
existing_ids = ActiveRecord::Base.connection.execute(
|
378
|
+
"SELECT #{afk} FROM #{ajt} WHERE #{fk} = #{base_id}"
|
379
|
+
).map { |r| r[afk] }
|
380
|
+
new_or_existing = assoc.klass.find_or_create_by(criteria)
|
381
|
+
new_or_existing_id = new_or_existing.send(new_or_existing.class.primary_key)
|
382
|
+
unless existing_ids.include?(new_or_existing_id)
|
383
|
+
ActiveRecord::Base.connection.execute(
|
384
|
+
"INSERT INTO #{ajt} (#{fk}, #{afk}) VALUES (#{base_id}, #{new_or_existing_id})"
|
385
|
+
)
|
386
|
+
end
|
387
|
+
new_or_existing
|
388
|
+
else # Must be a has_many
|
389
|
+
assoc.klass.find_or_create_by(criteria.merge({ assoc.foreign_key => base_id }))
|
390
|
+
end
|
391
|
+
else
|
392
|
+
klass_or_collection.find_by(criteria)
|
393
|
+
end
|
672
394
|
# If not successful, such as when fields are exposed via helper methods instead of being
|
673
395
|
# real columns in the database tables, try this more intensive approach. This is useful
|
674
396
|
# if you had full name kind of data coming in on a spreadsheeet, but in the destination
|
@@ -691,59 +413,453 @@ module DutyFree
|
|
691
413
|
# Standard criteria as well as foreign key column name detail with exact foreign keys
|
692
414
|
# that match up to a primary key so that if needed a new related object can be built,
|
693
415
|
# complete with all its association detail.
|
694
|
-
[found_object, criteria
|
416
|
+
[found_object, criteria, bt_criteria]
|
417
|
+
end # _find_existing
|
418
|
+
end # module ClassMethods
|
419
|
+
|
420
|
+
# With an array of incoming data, the first row having column names, perform the import
|
421
|
+
def self.import(obj_klass, data, import_template = nil, insert_only)
|
422
|
+
instance_variable_set(:@defined_uniques, nil)
|
423
|
+
instance_variable_set(:@valid_uniques, nil)
|
424
|
+
|
425
|
+
import_template ||= if obj_klass.constants.include?(:IMPORT_TEMPLATE)
|
426
|
+
obj_klass::IMPORT_TEMPLATE
|
427
|
+
else
|
428
|
+
obj_klass.suggest_template(0, false, false)
|
429
|
+
end
|
430
|
+
# puts "Chose #{import_template}"
|
431
|
+
inserts = []
|
432
|
+
updates = []
|
433
|
+
counts = Hash.new { |h, k| h[k] = [] }
|
434
|
+
errors = []
|
435
|
+
|
436
|
+
is_first = true
|
437
|
+
uniques = nil
|
438
|
+
cols = nil
|
439
|
+
starred = []
|
440
|
+
partials = []
|
441
|
+
# See if we can find the model if given only a string
|
442
|
+
if obj_klass.is_a?(String)
|
443
|
+
obj_klass_camelized = obj_klass.camelize.singularize
|
444
|
+
obj_klass = Object.const_get(obj_klass_camelized) if Object.const_defined?(obj_klass_camelized)
|
445
|
+
end
|
446
|
+
table_name = obj_klass.table_name unless obj_klass.is_a?(String)
|
447
|
+
is_build_table = if import_template.include?(:all!)
|
448
|
+
# Search for presence of this table
|
449
|
+
table_name = obj_klass.underscore.pluralize if obj_klass.is_a?(String)
|
450
|
+
!ActiveRecord::ConnectionAdapters::SchemaStatements.table_exists?(table_name)
|
451
|
+
else
|
452
|
+
false
|
453
|
+
end
|
454
|
+
# If we do need to build the table then we require an :all!, otherwise in the case that the table
|
455
|
+
# does not need to be built or was already built then try :all, and fall back on :all!.
|
456
|
+
all = import_template[is_build_table ? :all! : :all] || import_template[:all!]
|
457
|
+
keepers = {}
|
458
|
+
valid_unique = nil
|
459
|
+
existing = {}
|
460
|
+
devise_class = ''
|
461
|
+
ret = nil
|
462
|
+
|
463
|
+
# Multi-tenancy gem Apartment can be used if there are separate schemas per tenant
|
464
|
+
reference_models = if Object.const_defined?('Apartment')
|
465
|
+
Apartment.excluded_models
|
466
|
+
else
|
467
|
+
[]
|
468
|
+
end
|
469
|
+
|
470
|
+
if Object.const_defined?('Devise')
|
471
|
+
Object.const_get('Devise') # If this fails, devise_class will remain a blank string.
|
472
|
+
devise_class = Devise.mappings.values.first.class_name
|
473
|
+
reference_models -= [devise_class]
|
474
|
+
else
|
475
|
+
devise_class = ''
|
695
476
|
end
|
696
477
|
|
697
|
-
|
478
|
+
# Did they give us a filename?
|
479
|
+
if data.is_a?(String)
|
480
|
+
# Filenames with full paths can not be longer than 4096 characters, and can not
|
481
|
+
# include newline characters
|
482
|
+
data = if data.length <= 4096 && !data.index('\n')
|
483
|
+
File.open(data)
|
484
|
+
else
|
485
|
+
# Any multi-line string is likely CSV data
|
486
|
+
# %%% Test to see if TAB characters are present on the first line, instead of commas
|
487
|
+
CSV.new(data)
|
488
|
+
end
|
489
|
+
end
|
490
|
+
# Or perhaps us a file?
|
491
|
+
if data.is_a?(File)
|
492
|
+
# Use the "roo" gem if it's available
|
493
|
+
# When we're ready to try parsing this thing on our own, shared strings and all, then use
|
494
|
+
# the rubyzip gem also along with it:
|
495
|
+
# https://github.com/rubyzip/rubyzip
|
496
|
+
# require 'zip'
|
497
|
+
data = if Object.const_defined?('Roo::Spreadsheet', { csv_options: { encoding: 'bom|utf-8' } })
|
498
|
+
Roo::Spreadsheet.open(data)
|
499
|
+
else
|
500
|
+
# Otherwise generic CSV parsing
|
501
|
+
require 'csv' unless Object.const_defined?('CSV')
|
502
|
+
CSV.open(data)
|
503
|
+
end
|
504
|
+
end
|
698
505
|
|
699
|
-
|
700
|
-
|
701
|
-
|
702
|
-
|
703
|
-
|
704
|
-
|
705
|
-
|
706
|
-
|
707
|
-
|
708
|
-
|
709
|
-
|
710
|
-
|
506
|
+
# Will show as just one transaction when using auditing solutions such as PaperTrail
|
507
|
+
ActiveRecord::Base.transaction do
|
508
|
+
# Check to see if they want to do anything before the whole import
|
509
|
+
# First if defined in the import_template, then if there is a method in the class,
|
510
|
+
# and finally (not yet implemented) a generic global before_import
|
511
|
+
my_before_import = import_template[:before_import]
|
512
|
+
my_before_import ||= respond_to?(:before_import) && method(:before_import)
|
513
|
+
# my_before_import ||= some generic my_before_import
|
514
|
+
if my_before_import
|
515
|
+
last_arg_idx = my_before_import.parameters.length - 1
|
516
|
+
arguments = [data, import_template][0..last_arg_idx]
|
517
|
+
data = ret if (ret = my_before_import.call(*arguments)).is_a?(Enumerable)
|
518
|
+
end
|
519
|
+
build_tables = nil
|
520
|
+
data.each_with_index do |row, row_num|
|
521
|
+
row_errors = {}
|
522
|
+
if is_first # Anticipate that first row has column names
|
523
|
+
uniques = import_template[:uniques]
|
524
|
+
|
525
|
+
# Look for UTF-8 BOM in very first cell
|
526
|
+
row[0] = row[0][3..-1] if row[0].start_with?([239, 187, 191].pack('U*'))
|
527
|
+
# How about a first character of FEFF or FFFE to support UTF-16 BOMs?
|
528
|
+
# FE FF big-endian (standard)
|
529
|
+
# FF FE little-endian
|
530
|
+
row[0] = row[0][1..-1] if [65_279, 65_534].include?(row[0][0].ord)
|
531
|
+
cols = row.map { |col| (col || '').strip }
|
532
|
+
|
533
|
+
# Unique column combinations can be called out explicitly in models using uniques: {...}, or to just
|
534
|
+
# define one column at a time simply mark with an asterisk.
|
535
|
+
# Track and clean up stars
|
536
|
+
starred = cols.select do |col|
|
537
|
+
if col[0] == '*'
|
538
|
+
col.slice!(0)
|
539
|
+
col.strip!
|
540
|
+
end
|
541
|
+
end
|
542
|
+
partials = cols.select do |col|
|
543
|
+
if col[0] == '~'
|
544
|
+
col.slice!(0)
|
545
|
+
col.strip!
|
546
|
+
end
|
547
|
+
end
|
548
|
+
cols.map! { |col| ::DutyFree::Util._clean_name(col, import_template[:as]) }
|
549
|
+
obj_klass.send(:_defined_uniques, uniques, cols, cols.join('|'), starred)
|
550
|
+
# Main object asking for table to be built?
|
551
|
+
build_tables = {}
|
552
|
+
build_tables[path_name] = [namespaces, is_need_model, is_need_table] if is_build_table
|
553
|
+
# Make sure that at least half of them match what we know as being good column names
|
554
|
+
template_column_objects = ::DutyFree::Extensions._recurse_def(obj_klass, import_template[:all], import_template, build_tables).first
|
555
|
+
cols.each_with_index do |col, idx|
|
556
|
+
# prefixes = col_detail.pre_prefix + (col_detail.prefix.blank? ? [] : [col_detail.prefix])
|
557
|
+
# %%% Would be great here if when one comes back nil, try to find the closest match
|
558
|
+
# and indicate to the user a "did you mean?" about it.
|
559
|
+
keepers[idx] = template_column_objects.find { |col_obj| col_obj.titleize == col }
|
560
|
+
# puts "Could not find a match for column #{idx + 1}, #{col}" if keepers[idx].nil?
|
561
|
+
end
|
562
|
+
raise ::DutyFree::LessThanHalfAreMatchingColumnsError, I18n.t('import.altered_import_template_coumns') if keepers.length < (cols.length / 2) - 1
|
563
|
+
|
564
|
+
# Returns just the first valid unique lookup set if there are multiple
|
565
|
+
valid_unique, bt_criteria = obj_klass.send(:_find_existing, uniques, cols, starred, import_template, keepers, false, insert_only)
|
566
|
+
# Make a lookup from unique values to specific IDs
|
567
|
+
existing = obj_klass.pluck(*([:id] + valid_unique.keys)).each_with_object(existing) do |v, s|
|
568
|
+
s[v[1..-1].map(&:to_s)] = v.first
|
569
|
+
s
|
570
|
+
end
|
571
|
+
is_first = false
|
572
|
+
else # Normal row of data
|
573
|
+
is_insert = false
|
574
|
+
existing_unique = valid_unique.inject([]) do |s, v|
|
575
|
+
s << if v.last.is_a?(Array)
|
576
|
+
v.last[0].where(v.last[1] => row[v.last[2]]).limit(1).pluck(MAX_ID).first.to_s
|
577
|
+
else
|
578
|
+
row[v.last].to_s
|
579
|
+
end
|
580
|
+
end
|
581
|
+
to_be_saved = []
|
582
|
+
# Check to see if they want to preprocess anything
|
583
|
+
existing_unique = @before_process.call(valid_unique, existing_unique) if @before_process ||= import_template[:before_process]
|
584
|
+
obj = if (criteria = existing[existing_unique])
|
585
|
+
obj_klass.find(criteria)
|
586
|
+
else
|
587
|
+
is_insert = true
|
588
|
+
# unless build_tables.empty? # include?()
|
589
|
+
# binding.pry
|
590
|
+
# x = 5
|
591
|
+
# end
|
592
|
+
obj_klass.new
|
593
|
+
end
|
594
|
+
to_be_saved << [obj] unless criteria # || this one has any belongs_to that will be modified here
|
595
|
+
sub_obj = nil
|
596
|
+
polymorphics = []
|
597
|
+
sub_objects = {}
|
598
|
+
this_path = nil
|
599
|
+
keepers.each do |key, v|
|
600
|
+
next if v.nil?
|
601
|
+
|
602
|
+
sub_obj = obj
|
603
|
+
this_path = +''
|
604
|
+
# puts "p: #{v.path}"
|
605
|
+
v.path.each_with_index do |path_part, idx|
|
606
|
+
this_path << (this_path.blank? ? path_part.to_s : ",#{path_part}")
|
607
|
+
unless (sub_next = sub_objects[this_path])
|
608
|
+
# Check if we're hitting reference data / a lookup thing
|
609
|
+
assoc = v.prefix_assocs[idx]
|
610
|
+
# belongs_to some lookup (reference) data
|
611
|
+
if assoc && reference_models.include?(assoc.class_name)
|
612
|
+
lookup_match = assoc.klass.find_by(v.name => row[key])
|
613
|
+
# Do a partial match if this column allows for it
|
614
|
+
# and we only find one matching result.
|
615
|
+
if lookup_match.nil? && partials.include?(v.titleize)
|
616
|
+
lookup_match ||= assoc.klass.where("#{v.name} LIKE '#{row[key]}%'")
|
617
|
+
lookup_match = (lookup_match.length == 1 ? lookup_match.first : nil)
|
618
|
+
end
|
619
|
+
sub_obj.send("#{path_part}=", lookup_match) unless lookup_match.nil?
|
620
|
+
# Reference data from the public level means we stop here
|
621
|
+
sub_obj = nil
|
622
|
+
break
|
623
|
+
end
|
624
|
+
# Get existing related object, or create a new one
|
625
|
+
# This first part works for belongs_to. has_many and has_one get sorted below.
|
626
|
+
# start = (v.pre_prefix.blank? ? 0 : v.pre_prefix.length)
|
627
|
+
start = 0
|
628
|
+
trim_prefix = v.titleize[start..-(v.name.length + 2)]
|
629
|
+
trim_prefix << ' ' unless trim_prefix.blank?
|
630
|
+
binding.pry unless assoc
|
631
|
+
if assoc.belongs_to?
|
632
|
+
klass = Object.const_get(assoc&.class_name)
|
633
|
+
# Try to find a unique item if one is referenced
|
634
|
+
sub_next = nil
|
635
|
+
begin
|
636
|
+
sub_next, criteria, bt_criteria = klass.send(:_find_existing, uniques, cols, starred, import_template, keepers, nil,
|
637
|
+
false, # insert_only
|
638
|
+
row, klass, all, trim_prefix, assoc)
|
639
|
+
rescue ::DutyFree::NoUniqueColumnError
|
640
|
+
end
|
641
|
+
bt_name = "#{path_part}="
|
642
|
+
# Not yet wired up to the right one, or going to the parent of a self-referencing model?
|
643
|
+
# puts "#{v.path} #{criteria.inspect}"
|
644
|
+
unless sub_next || (klass == sub_obj.class && (all_criteria = criteria.merge(bt_criteria)).empty?)
|
645
|
+
sub_next = klass.new(all_criteria || {})
|
646
|
+
to_be_saved << [sub_next, sub_obj, bt_name, sub_next]
|
647
|
+
end
|
648
|
+
# This wires it up in memory, but doesn't yet put the proper foreign key ID in
|
649
|
+
# place when the primary object is a new one (and thus not having yet been saved
|
650
|
+
# doesn't yet have an ID).
|
651
|
+
# binding.pry if !sub_next.new_record? && sub_next.name == 'Squidget Widget' # !sub_obj.changed?
|
652
|
+
# # %%% Question is -- does the above set changed? when a foreign key is not yet set
|
653
|
+
# # and only the in-memory object has changed?
|
654
|
+
is_yet_changed = sub_obj.changed?
|
655
|
+
sub_obj.send(bt_name, sub_next)
|
656
|
+
|
657
|
+
# We need this in the case of updating the primary object across a belongs_to when
|
658
|
+
# the foreign one already exists and is not otherwise changing, such as when it is
|
659
|
+
# not a new one, so wouldn't otherwise be getting saved.
|
660
|
+
to_be_saved << [sub_obj] if !sub_obj.new_record? && !is_yet_changed && sub_obj.changed?
|
661
|
+
# From a has_many or has_one?
|
662
|
+
# Rails 4.0 and later can do: sub_next.is_a?(ActiveRecord::Associations::CollectionProxy)
|
663
|
+
elsif [:has_many, :has_one, :has_and_belongs_to_many].include?(assoc.macro) # && !assoc.options[:through]
|
664
|
+
::DutyFree::Extensions._save_pending(to_be_saved)
|
665
|
+
sub_next = sub_obj.send(path_part)
|
666
|
+
# Try to find a unique item if one is referenced
|
667
|
+
# %%% There is possibility that when bringing in related classes using a nil
|
668
|
+
# in IMPORT_TEMPLATE[:all] that this will break. Need to test deeply nested things.
|
669
|
+
|
670
|
+
# assoc.inverse_of is the belongs_to side of the has_many train we came in here on.
|
671
|
+
sub_hm, criteria, bt_criteria = assoc.klass.send(:_find_existing, uniques, cols, starred, import_template, keepers, assoc.inverse_of,
|
672
|
+
false, # insert_only
|
673
|
+
row, sub_next, all, trim_prefix, assoc,
|
674
|
+
# Just in case we're running Rails < 4.0 and this is a has_*
|
675
|
+
sub_obj)
|
676
|
+
# If still not found then create a new related object using this has_many collection
|
677
|
+
# (criteria.empty? ? nil : sub_next.new(criteria))
|
678
|
+
if sub_hm
|
679
|
+
sub_next = sub_hm
|
680
|
+
elsif assoc.macro == :has_one
|
681
|
+
# assoc.active_record.name.underscore is only there to support older Rails
|
682
|
+
# that doesn't do automatic inverse_of
|
683
|
+
ho_name = "#{assoc.inverse_of&.name || assoc.active_record.name.underscore}="
|
684
|
+
sub_next = assoc.klass.new(criteria)
|
685
|
+
to_be_saved << [sub_next, sub_next, ho_name, sub_obj]
|
686
|
+
elsif assoc.macro == :has_and_belongs_to_many ||
|
687
|
+
(assoc.macro == :has_many && assoc.options[:through])
|
688
|
+
# sub_next = sub_next.new(criteria)
|
689
|
+
# Search for one to wire up if it might already exist, otherwise create one
|
690
|
+
sub_next = assoc.klass.find_by(criteria) || assoc.klass.new(criteria)
|
691
|
+
to_be_saved << [sub_next, :has_and_belongs_to_many, assoc.name, sub_obj]
|
692
|
+
else
|
693
|
+
# Two other methods that are possible to check for here are :conditions and
|
694
|
+
# :sanitized_conditions, which do not exist in Rails 4.0 and later.
|
695
|
+
sub_next = if assoc.respond_to?(:require_association)
|
696
|
+
# With Rails < 4.0, sub_next could come in as an Array or a broken CollectionProxy
|
697
|
+
assoc.klass.new({ ::DutyFree::Extensions._fk_from(assoc) => sub_obj.send(sub_obj.class.primary_key) }.merge(criteria))
|
698
|
+
else
|
699
|
+
sub_next.proxy_association.reflection.instance_variable_set(:@foreign_key, ::DutyFree::Extensions._fk_from(assoc))
|
700
|
+
sub_next.new(criteria)
|
701
|
+
end
|
702
|
+
to_be_saved << [sub_next]
|
703
|
+
end
|
704
|
+
# else
|
705
|
+
# belongs_to for a found object, or a HMT
|
706
|
+
end
|
707
|
+
# # Incompatible with Rails < 4.2
|
708
|
+
# # Look for possible missing polymorphic detail
|
709
|
+
# # Maybe can test for this via assoc.through_reflection
|
710
|
+
# if assoc.is_a?(ActiveRecord::Reflection::ThroughReflection) &&
|
711
|
+
# (delegate = assoc.send(:delegate_reflection)&.active_record&.reflect_on_association(assoc.source_reflection_name)) &&
|
712
|
+
# delegate.options[:polymorphic]
|
713
|
+
# polymorphics << { parent: sub_next, child: sub_obj, type_col: delegate.foreign_type, id_col: delegate.foreign_key.to_s }
|
714
|
+
# end
|
715
|
+
|
716
|
+
# rubocop:disable Style/SoleNestedConditional
|
717
|
+
unless sub_next.nil?
|
718
|
+
# if sub_next.class.name == devise_class && # only for Devise users
|
719
|
+
# sub_next.email =~ Devise.email_regexp
|
720
|
+
# if existing.include?([sub_next.email])
|
721
|
+
# User already exists
|
722
|
+
# else
|
723
|
+
# sub_next.invite!
|
724
|
+
# end
|
725
|
+
# end
|
726
|
+
sub_objects[this_path] = sub_next if this_path.present?
|
727
|
+
end
|
728
|
+
# rubocop:enable Style/SoleNestedConditional
|
729
|
+
end
|
730
|
+
sub_obj = sub_next
|
731
|
+
end
|
732
|
+
next if sub_obj.nil?
|
711
733
|
|
712
|
-
|
713
|
-
|
734
|
+
next unless sub_obj.respond_to?(sym = "#{v.name}=".to_sym)
|
735
|
+
|
736
|
+
col_type = sub_obj.class.columns_hash[v.name.to_s]&.type
|
737
|
+
if col_type.nil? && (virtual_columns = import_template[:virtual_columns]) &&
|
738
|
+
(virtual_columns = virtual_columns[this_path] || virtual_columns)
|
739
|
+
col_type = virtual_columns[v.name]
|
714
740
|
end
|
715
|
-
|
716
|
-
|
717
|
-
|
741
|
+
if col_type == :boolean
|
742
|
+
if row[key].nil?
|
743
|
+
# Do nothing when it's nil
|
744
|
+
elsif %w[true t yes y].include?(row[key]&.strip&.downcase) # Used to cover 'on'
|
745
|
+
row[key] = true
|
746
|
+
elsif %w[false f no n].include?(row[key]&.strip&.downcase) # Used to cover 'off'
|
747
|
+
row[key] = false
|
748
|
+
else
|
749
|
+
row_errors[v.name] ||= []
|
750
|
+
row_errors[v.name] << "Boolean value \"#{row[key]}\" in column #{key + 1} not recognized"
|
751
|
+
end
|
718
752
|
end
|
719
|
-
|
720
|
-
|
721
|
-
|
722
|
-
|
723
|
-
|
724
|
-
|
753
|
+
|
754
|
+
is_yet_changed = sub_obj.changed?
|
755
|
+
sub_obj.send(sym, row[key])
|
756
|
+
# If this one is transitioning from having not changed to now having been changed,
|
757
|
+
# and is not a new one anyway that would already be lined up to get saved, then we
|
758
|
+
# mark it to now be saved.
|
759
|
+
to_be_saved << [sub_obj] if !sub_obj.new_record? && !is_yet_changed && sub_obj.changed?
|
760
|
+
# else
|
761
|
+
# puts " #{sub_obj.class.name} doesn't respond to #{sym}"
|
762
|
+
end
|
763
|
+
# Try to save final sub-object(s) if any exist
|
764
|
+
::DutyFree::Extensions._save_pending(to_be_saved)
|
765
|
+
|
766
|
+
# Reinstate any missing polymorphic _type and _id values
|
767
|
+
polymorphics.each do |poly|
|
768
|
+
if !poly[:parent].new_record? || poly[:parent].save
|
769
|
+
poly[:child].send("#{poly[:type_col]}=".to_sym, poly[:parent].class.name)
|
770
|
+
poly[:child].send("#{poly[:id_col]}=".to_sym, poly[:parent].id)
|
725
771
|
end
|
726
772
|
end
|
727
|
-
|
728
|
-
|
729
|
-
|
730
|
-
|
731
|
-
|
773
|
+
|
774
|
+
# Give a window of opportunity to tweak user objects controlled by Devise
|
775
|
+
obj_class = obj.class
|
776
|
+
is_do_save = if obj_class.respond_to?(:before_devise_save) && obj_class.name == devise_class
|
777
|
+
obj_class.before_devise_save(obj, existing)
|
778
|
+
else
|
779
|
+
true
|
780
|
+
end
|
781
|
+
|
782
|
+
if obj.valid?
|
783
|
+
obj.save if is_do_save
|
784
|
+
# Snag back any changes to the unique columns. (For instance, Devise downcases email addresses.)
|
785
|
+
existing_unique = valid_unique.keys.inject([]) { |s, v| s << obj.send(v).to_s }
|
786
|
+
# Update the duplicate counts and inserted / updated results
|
787
|
+
counts[existing_unique] << row_num
|
788
|
+
(is_insert ? inserts : updates) << { row_num => existing_unique } if is_do_save
|
789
|
+
# Track this new object so we can properly sense any duplicates later
|
790
|
+
existing[existing_unique] = obj.id
|
791
|
+
else
|
792
|
+
row_errors.merge! obj.errors.messages
|
793
|
+
end
|
794
|
+
errors << { row_num => row_errors } unless row_errors.empty?
|
732
795
|
end
|
733
|
-
@defined_uniques[col_list] = defined_uniq
|
734
796
|
end
|
735
|
-
|
797
|
+
duplicates = counts.each_with_object([]) do |v, s|
|
798
|
+
s += v.last[1..-1].map { |line_num| { line_num => v.first } } if v.last.count > 1
|
799
|
+
s
|
800
|
+
end
|
801
|
+
ret = { inserted: inserts, updated: updates, duplicates: duplicates, errors: errors }
|
802
|
+
|
803
|
+
# Check to see if they want to do anything after the import
|
804
|
+
# First if defined in the import_template, then if there is a method in the class,
|
805
|
+
# and finally (not yet implemented) a generic global after_import
|
806
|
+
my_after_import = import_template[:after_import]
|
807
|
+
my_after_import ||= respond_to?(:after_import) && method(:after_import)
|
808
|
+
# my_after_import ||= some generic my_after_import
|
809
|
+
if my_after_import
|
810
|
+
last_arg_idx = my_after_import.parameters.length - 1
|
811
|
+
arguments = [ret][0..last_arg_idx]
|
812
|
+
# rubocop:disable Lint/UselessAssignment
|
813
|
+
ret = ret2 if (ret2 = my_after_import.call(*arguments)).is_a?(Hash)
|
814
|
+
# rubocop:enable Lint/UselessAssignment
|
815
|
+
end
|
736
816
|
end
|
737
|
-
|
817
|
+
ret
|
818
|
+
end
|
819
|
+
|
820
|
+
def self._fk_from(assoc)
|
821
|
+
# Try first to trust whatever they've marked as being the foreign_key, and then look
|
822
|
+
# at the inverse's foreign key setting if available. In all cases don't accept
|
823
|
+
# anything that's not backed with a real column in the table.
|
824
|
+
col_names = assoc.klass.column_names
|
825
|
+
if (
|
826
|
+
(fk_name = assoc.options[:foreign_key]) ||
|
827
|
+
(fk_name = assoc.inverse_of&.options&.fetch(:foreign_key) { nil }) ||
|
828
|
+
(assoc.respond_to?(:foreign_key) && (fk_name = assoc.foreign_key)) ||
|
829
|
+
(fk_name = assoc.inverse_of&.foreign_key) ||
|
830
|
+
(fk_name = assoc.inverse_of&.association_foreign_key)
|
831
|
+
) && col_names.include?(fk_name.to_s)
|
832
|
+
return fk_name
|
833
|
+
end
|
834
|
+
|
835
|
+
# Don't let this fool you -- we really are in search of the foreign key name here,
|
836
|
+
# and Rails 3.0 and older used some fairly interesting conventions, calling it instead
|
837
|
+
# the "primary_key_name"!
|
838
|
+
if assoc.respond_to?(:primary_key_name)
|
839
|
+
if (fk_name = assoc.primary_key_name) && col_names.include?(fk_name.to_s)
|
840
|
+
return fk_name
|
841
|
+
end
|
842
|
+
if (fk_name = assoc.inverse_of.primary_key_name) && col_names.include?(fk_name.to_s)
|
843
|
+
return fk_name
|
844
|
+
end
|
845
|
+
end
|
846
|
+
|
847
|
+
puts "* Wow, had no luck at all finding a foreign key for #{assoc.inspect}"
|
848
|
+
end
|
849
|
+
|
850
|
+
# unless build_tables.include?((foreign_class = tbs[2].class).underscore)
|
851
|
+
# build_table(build_tables, tbs[1].class, foreign_class, :has_one)
|
738
852
|
|
739
853
|
# Called before building any object linked through a has_one or has_many so that foreign key IDs
|
740
854
|
# can be added properly to those new objects. Finally at the end also called to save everything.
|
741
855
|
def self._save_pending(to_be_saved)
|
742
856
|
while (tbs = to_be_saved.pop)
|
743
857
|
# puts "Will be calling #{tbs[1].class.name} #{tbs[1]&.id} .#{tbs[2]} #{tbs[3].class.name} #{tbs[3]&.id}"
|
744
|
-
|
745
|
-
|
746
|
-
|
858
|
+
# Wire this one up if it had come from a has_one
|
859
|
+
if tbs[0] == tbs[1]
|
860
|
+
tbs[1].class.has_one(tbs[2]) unless tbs[1].respond_to?(tbs[2])
|
861
|
+
tbs[1].send(tbs[2], tbs[3])
|
862
|
+
end
|
747
863
|
|
748
864
|
ais = (tbs.first.class.respond_to?(:around_import_save) && tbs.first.class.method(:around_import_save)) ||
|
749
865
|
(respond_to?(:around_import_save) && method(:around_import_save))
|
@@ -764,7 +880,17 @@ module DutyFree
|
|
764
880
|
tbs[0] == tbs[1] || # From a has_one?
|
765
881
|
tbs.first.new_record?
|
766
882
|
|
767
|
-
tbs[1]
|
883
|
+
if tbs[1] == :has_and_belongs_to_many # also used by has_many :through associations
|
884
|
+
collection = tbs[3].send(tbs[2])
|
885
|
+
being_shoveled_id = tbs[0].send(tbs[0].class.primary_key)
|
886
|
+
if collection.empty? ||
|
887
|
+
!collection.pluck("#{(klass = collection.first.class).table_name}.#{klass.primary_key}").include?(being_shoveled_id)
|
888
|
+
collection << tbs[0]
|
889
|
+
# puts collection.inspect
|
890
|
+
end
|
891
|
+
else # Traditional belongs_to
|
892
|
+
tbs[1].send(tbs[2], tbs[3])
|
893
|
+
end
|
768
894
|
end
|
769
895
|
end
|
770
896
|
|
@@ -783,40 +909,71 @@ module DutyFree
|
|
783
909
|
template_detail_columns
|
784
910
|
end
|
785
911
|
|
786
|
-
# Recurse and return
|
787
|
-
# nested hashes to be used with ActiveRecord's .joins() to facilitate export
|
788
|
-
|
912
|
+
# Recurse and return three arrays -- one with all columns in sequence, and one a hierarchy of
|
913
|
+
# nested hashes to be used with ActiveRecord's .joins() to facilitate export, and finally
|
914
|
+
# one that lists tables that need to be built along the way.
|
915
|
+
def self._recurse_def(klass, cols, import_template, build_tables = nil, order_by = nil, assocs = [], joins = [], pre_prefix = '', prefix = '')
|
916
|
+
prefix = prefix[0..-2] if (is_build_table = prefix.end_with?('!'))
|
789
917
|
prefixes = ::DutyFree::Util._prefix_join([pre_prefix, prefix])
|
790
|
-
|
791
|
-
|
792
|
-
|
793
|
-
|
794
|
-
|
795
|
-
|
918
|
+
|
919
|
+
if prefix.present?
|
920
|
+
# An indication to build this table and model if it doesn't exist?
|
921
|
+
if is_build_table && build_tables
|
922
|
+
namespaces = prefix.split('::')
|
923
|
+
model_name = namespaces.map { |part| part.singularize.camelize }.join('::')
|
924
|
+
prefix = namespaces.last
|
925
|
+
# %%% If the model already exists, take belongs_to cues from it for building the table
|
926
|
+
if (is_need_model = Object.const_defined?(model_name)) &&
|
927
|
+
(is_need_table = ActiveRecord::ConnectionAdapters::SchemaStatements.table_exists?(
|
928
|
+
(path_name = ::DutyFree::Util._prefix_join([prefixes, prefix.pluralize]))
|
929
|
+
))
|
930
|
+
is_build_table = false
|
931
|
+
else
|
932
|
+
build_tables[path_name] = [namespaces, is_need_model, is_need_table] unless build_tables.include?(path_name)
|
933
|
+
end
|
934
|
+
end
|
935
|
+
unless is_build_table && build_tables
|
936
|
+
# Confirm we can actually navigate through this association
|
937
|
+
prefix_assoc = (assocs.last&.klass || klass).reflect_on_association(prefix) if prefix.present?
|
938
|
+
if prefix_assoc
|
939
|
+
assocs = assocs.dup << prefix_assoc
|
940
|
+
if order_by && [:has_many, :has_and_belongs_to_many].include?(prefix_assoc.macro) &&
|
941
|
+
(pk = prefix_assoc.active_record.primary_key)
|
942
|
+
order_by << ["#{prefixes.tr('.', '_')}_", pk]
|
943
|
+
end
|
944
|
+
end
|
796
945
|
end
|
797
946
|
end
|
798
|
-
|
947
|
+
cols = cols.inject([]) do |s, col|
|
799
948
|
s + if col.is_a?(Hash)
|
800
949
|
col.inject([]) do |s2, v|
|
801
|
-
|
802
|
-
|
950
|
+
if order_by
|
951
|
+
# Find what the type is for this guy
|
952
|
+
next_klass = (assocs.last&.klass || klass).reflect_on_association(v.first)&.klass
|
953
|
+
# Used to be: { v.first.to_sym => (joins_array = []) }
|
954
|
+
joins << { v.first.to_sym => (joins_array = [next_klass]) }
|
955
|
+
end
|
956
|
+
s2 + _recurse_def(klass, (v.last.is_a?(Array) ? v.last : [v.last]), import_template, build_tables, order_by, assocs, joins_array, prefixes, v.first.to_sym).first
|
803
957
|
end
|
804
958
|
elsif col.nil?
|
805
959
|
if assocs.empty?
|
806
960
|
[]
|
807
961
|
else
|
808
962
|
# Bring in from another class
|
809
|
-
|
963
|
+
# Used to be: { prefix => (joins_array = []) }
|
964
|
+
# %%% Probably need a next_klass thing like above
|
965
|
+
joins << { prefix => (joins_array = [klass]) } if order_by
|
810
966
|
# %%% Also bring in uniques and requireds
|
811
|
-
_recurse_def(klass, assocs.last.klass::IMPORT_TEMPLATE[:all], import_template, order_by, assocs, joins_array, prefixes).first
|
967
|
+
_recurse_def(klass, assocs.last.klass::IMPORT_TEMPLATE[:all], import_template, build_tables, order_by, assocs, joins_array, prefixes).first
|
812
968
|
end
|
813
969
|
else
|
814
970
|
[::DutyFree::Column.new(col, pre_prefix, prefix, assocs, klass, import_template[:as])]
|
815
971
|
end
|
816
972
|
end
|
817
|
-
[
|
973
|
+
[cols, joins]
|
818
974
|
end
|
819
975
|
end # module Extensions
|
976
|
+
# rubocop:enable Style/CommentedKeyword
|
820
977
|
|
821
978
|
# Rails < 4.0 doesn't have ActiveRecord::RecordNotUnique, so use the more generic ActiveRecord::ActiveRecordError instead
|
822
979
|
ar_not_unique_error = ActiveRecord.const_defined?('RecordNotUnique') ? ActiveRecord::RecordNotUnique : ActiveRecord::ActiveRecordError
|