duty_free 1.0.5 → 1.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 51bb54397528578ae7d4d43a04d1d6c31f81d1417d9c076f1fb751b18c09afaa
-  data.tar.gz: 8561d19c0a0166462c011f6a310429944de9b95751dbf9b36dbbb32fdce4bf76
+  metadata.gz: f9ac922dd807fc18539ff130c1ccf63412e837acbe4eba0127423235c11d0f1e
+  data.tar.gz: e5c27fe4a58718dcbbd4c5ebc1aed2dbb5e7e4b386c977e0ec31269cfad08d48
 SHA512:
-  metadata.gz: 02ae8681512885e5ab618aaef343ae88375338fc396ec711b28db78bd97daad361949b86c18c1ca81401e3a15f6629814900a751145bf8a270e50f1fa5669cab
-  data.tar.gz: 0444426a5468f3dcf7143f3f8859f79f2808db968b6a59561ff0351223993b0071aa94e95b4e50924d99ccbf4ae78f7a9b190a617eac6c21305d8ae0e7dd7f87
+  metadata.gz: 54811a68bd34fd5bb881fa19006a7297a8697a3a1575b7060ba7d82e6fcd6d36f545e1d5cfcfc51c756a5b87880e445c415924d2f54aa39471f0b884f0b4c6c4
+  data.tar.gz: 32ccb88e429e64bccb59c7741955b99537c661d6416d6f8c352cb384df728081d5b54079cb2dbd904203fbcf1c60abb1a4a7c70bee9c2764fa1622eaaeb6843f

data/lib/duty_free.rb

@@ -1,5 +1,47 @@
 # frozen_string_literal: true
 
+require 'active_record/version'
+
+# ActiveRecord before 4.0 didn't have #version
+unless ActiveRecord.respond_to?(:version)
+  module ActiveRecord
+    def self.version
+      ::Gem::Version.new(ActiveRecord::VERSION::STRING)
+    end
+  end
+end
+
+# In ActiveSupport older than 5.0, the duplicable? test tries to new up a BigDecimal,
+# and Ruby 2.6 and later deprecates #new.  This removes the warning from BigDecimal.
+require 'bigdecimal'
+if ActiveRecord.version < ::Gem::Version.new('5.0') &&
+   ::Gem::Version.new(RUBY_VERSION) >= ::Gem::Version.new('2.6')
+  def BigDecimal.new(*args, **kwargs)
+    BigDecimal(*args, **kwargs)
+  end
+end
+
+# Allow Rails 4.0 and 4.1 to work with newer Ruby (>= 2.4) by avoiding a "stack level too deep" error
+# when ActiveSupport tries to smarten up Numeric by messing with Fixnum and Bignum at the end of:
+# activesupport-4.0.13/lib/active_support/core_ext/numeric/conversions.rb
+if ActiveRecord.version < ::Gem::Version.new('4.2') &&
+   ActiveRecord.version > ::Gem::Version.new('3.2') &&
+   Object.const_defined?('Integer') && Integer.superclass.name == 'Numeric'
+  class OurFixnum < Integer; end
+  Numeric.const_set('Fixnum', OurFixnum)
+  class OurBignum < Integer; end
+  Numeric.const_set('Bignum', OurBignum)
+end
+
+# Allow Rails < 3.2 to run with newer versions of Psych gem
+if BigDecimal.respond_to?(:yaml_tag) && !BigDecimal.respond_to?(:yaml_as)
+  class BigDecimal
+    class <<self
+      alias yaml_as yaml_tag
+    end
+  end
+end
+
 require 'active_record'
 
 require 'duty_free/config'
@@ -65,15 +107,6 @@ end
 # Major compatibility fixes for ActiveRecord < 4.2
 # ================================================
 ActiveSupport.on_load(:active_record) do
-  # ActiveRecord before 4.0 didn't have #version
-  unless ActiveRecord.respond_to?(:version)
-    module ActiveRecord
-      def self.version
-        ::Gem::Version.new(ActiveRecord::VERSION::STRING)
-      end
-    end
-  end
-
   # Rails < 4.0 cannot do #find_by, or do #pluck on multiple columns, so here are the patches:
   if ActiveRecord.version < ::Gem::Version.new('4.0')
     module ActiveRecord
@@ -97,7 +130,18 @@ ActiveSupport.on_load(:active_record) do
           else
             relation = clone # spawn
             relation.select_values = column_names
-            result = if respond_to?(:bind_values)
+            result = if klass.connection.class.name.end_with?('::PostgreSQLAdapter')
+                       rslt = klass.connection.execute(relation.arel.to_sql)
+                       rslt.type_map =
+                         @type_map ||= proc do
+                           # This aliasing avoids the warning:
+                           # "no type cast defined for type "numeric" with oid 1700. Please cast this type
+                           # explicitly to TEXT to be safe for future changes."
+                           PG::BasicTypeRegistry.alias_type(0, 'numeric', 'text')
+                           PG::BasicTypeMapForResults.new(klass.connection.raw_connection)
+                         end.call
+                       rslt.to_a
+                     elsif respond_to?(:bind_values)
                        klass.connection.select_all(relation.arel, nil, bind_values)
                      else
                        klass.connection.select_all(relation.arel.to_sql, nil)
@@ -193,6 +237,13 @@ ActiveSupport.on_load(:active_record) do
     end
   end
 
+  if ActiveRecord.version < ::Gem::Version.new('5.0')
+    # Avoid pg gem deprecation warning:  "You should use PG::Connection, PG::Result, and PG::Error instead"
+    PGconn = PG::Connection
+    PGresult = PG::Result
+    PGError = PG::Error
+  end
+
   include ::DutyFree::Extensions
 end
 

data/lib/duty_free/extensions.rb

@@ -51,8 +51,10 @@ module DutyFree
         rows = [rows]
 
         if is_with_data
+          order_by = []
+          order_by << ['_', primary_key] if primary_key
           # Automatically create a JOINs strategy and select list to get back all related rows
-          template_cols, template_joins = ::DutyFree::Extensions._recurse_def(self, import_template[:all], import_template)
+          template_cols, template_joins = ::DutyFree::Extensions._recurse_def(self, import_template[:all], import_template, order_by)
           relation = use_inner_joins ? joins(template_joins) : left_joins(template_joins)
 
           # So we can properly create the SELECT list, create a mapping between our
@@ -68,7 +70,9 @@ module DutyFree
                              end
           our_names = ['_'] + ::DutyFree::Util._recurse_arel(template_joins)
           mapping = our_names.zip(arel_alias_names).to_h
-
+          relation = (order_by.empty? ? relation : relation.order(order_by.map { |o| "#{mapping[o.first]}.#{o.last}" }))
+          # puts mapping.inspect
+          # puts relation.dup.select(template_cols.map { |x| x.to_s(mapping) }).to_sql
           relation.select(template_cols.map { |x| x.to_s(mapping) }).each do |result|
             rows << ::DutyFree::Extensions._template_columns(self, import_template).map do |col|
               value = result.send(col)
@@ -217,20 +221,20 @@ module DutyFree
               is_insert = false
               existing_unique = valid_unique.inject([]) do |s, v|
                 s << if v.last.is_a?(Array)
-                       # binding.pry
                        v.last[0].where(v.last[1] => row[v.last[2]]).limit(1).pluck(MAX_ID).first.to_s
                      else
                        row[v.last].to_s
                      end
               end
+              to_be_saved = []
               # Check to see if they want to preprocess anything
               existing_unique = @before_process.call(valid_unique, existing_unique) if @before_process ||= import_template[:before_process]
-              obj = if existing.include?(existing_unique)
-                      find(existing[existing_unique])
-                    else
-                      is_insert = true
-                      new
-                    end
+              if (criteria = existing[existing_unique])
+                obj = find(criteria)
+              else
+                is_insert = true
+                to_be_saved << [obj = new]
+              end
               sub_obj = nil
               is_has_one = false
               has_ones = []
@@ -240,28 +244,13 @@ module DutyFree
               keepers.each do |key, v|
                 next if v.nil?
 
-                # Not the same as the last path?
-                if this_path && v.path != this_path.split(',').map(&:to_sym) && !is_has_one
-                  # puts sub_obj.class.name
-                  if respond_to?(:around_import_save)
-                    # Send them the sub_obj even if it might be invalid so they can choose
-                    # to make it valid if they wish.
-                    # binding.pry
-                    around_import_save(sub_obj) do |modded_obj = nil|
-                      modded_obj = (modded_obj || sub_obj)
-                      modded_obj.save if sub_obj&.valid?
-                    end
-                  elsif sub_obj&.valid?
-                    sub_obj.save
-                  end
-                end
                 sub_obj = obj
                 this_path = +''
                 # puts "p: #{v.path}"
                 v.path.each_with_index do |path_part, idx|
                   this_path << (this_path.blank? ? path_part.to_s : ",#{path_part}")
                   unless (sub_next = sub_objects[this_path])
-                    # Check if we're hitting platform data / a lookup thing
+                    # Check if we're hitting reference data / a lookup thing
                     assoc = v.prefix_assocs[idx]
                     # belongs_to some lookup (reference) data
                     if assoc && reference_models.include?(assoc.class_name)
@@ -273,44 +262,33 @@ module DutyFree
                         lookup_match = (lookup_match.length == 1 ? lookup_match.first : nil)
                       end
                       sub_obj.send("#{path_part}=", lookup_match) unless lookup_match.nil?
-                      # Reference data from the platform level means we stop here
+                      # Reference data from the public level means we stop here
                       sub_obj = nil
                       break
                     end
-                    # This works for belongs_to or has_one.  has_many gets sorted below.
                     # Get existing related object, or create a new one
-                    if (sub_next = sub_obj.send(path_part)).nil?
+                    # This first part works for belongs_to.  has_many and has_one get sorted below.
+                    if (sub_next = sub_obj.send(path_part)).nil? && assoc.belongs_to?
                       klass = Object.const_get(assoc&.class_name)
-                      # assoc.is_a?(ActiveRecord::Reflection::HasOneReflection)
-                      # %%% When we support only AR 4.2 and above then we can do:  assoc.has_one?
-                      sub_next = if assoc.macro == :has_one
-                                   has_ones << v.path
-                                   klass.new
-                                 else
-                                   # Try to find a unique item if one is referenced
-                                   sub_bt = nil
-                                   begin
-                                     trim_prefix = v.titleize[0..-(v.name.length + 2)]
-                                     trim_prefix << ' ' unless trim_prefix.blank?
-                                     sub_bt, criteria = klass.find_existing(uniques, cols, starred, import_template, keepers, nil, row, klass, all, trim_prefix)
-                                   rescue ::DutyFree::NoUniqueColumnError
-                                   end
-                                   # %%% Can criteria really ever be nil anymore?
-                                   sub_bt ||= klass.new(criteria || {}) unless klass == sub_obj.class && criteria.empty?
-                                   sub_obj.send("#{path_part}=", sub_bt)
-                                   sub_bt
-                                 end
-                    end
-                    # Look for possible missing polymorphic detail
-                    # Maybe can test for this via assoc.through_reflection
-                    if assoc.is_a?(ActiveRecord::Reflection::ThroughReflection) &&
-                       (delegate = assoc.send(:delegate_reflection)&.active_record&.reflect_on_association(assoc.source_reflection_name)) &&
-                       delegate.options[:polymorphic]
-                      polymorphics << { parent: sub_next, child: sub_obj, type_col: delegate.foreign_type, id_col: delegate.foreign_key.to_s }
-                    end
-                    # From a has_many?
+                      # Try to find a unique item if one is referenced
+                      sub_next = nil
+                      begin
+                        trim_prefix = v.titleize[0..-(v.name.length + 2)]
+                        trim_prefix << ' ' unless trim_prefix.blank?
+                        sub_next, criteria = klass.find_existing(uniques, cols, starred, import_template, keepers, nil, row, klass, all, trim_prefix)
+                      rescue ::DutyFree::NoUniqueColumnError
+                      end
+                      # puts "#{v.path} #{criteria.inspect}"
+                      bt_name = "#{path_part}="
+                      unless sub_next || (klass == sub_obj.class && criteria.empty?)
+                        sub_next = klass.new(criteria || {})
+                        to_be_saved << [sub_next, sub_obj, bt_name, sub_next]
+                      end
+                      sub_obj.send(bt_name, sub_next)
+                    # From a has_many or has_one?
                     # Rails 4.0 and later can do:  sub_next.is_a?(ActiveRecord::Associations::CollectionProxy)
-                    if assoc.macro == :has_many && !assoc.options[:through]
+                    elsif [:has_many, :has_one].include?(assoc.macro) && !assoc.options[:through]
+                      ::DutyFree::Extensions._save_pending(to_be_saved)
                       # Try to find a unique item if one is referenced
                       # %%% There is possibility that when bringing in related classes using a nil
                       # in IMPORT_TEMPLATE[:all] that this will break.  Need to test deeply nested things.
@@ -319,10 +297,33 @@ module DutyFree
                       trim_prefix << ' ' unless trim_prefix.blank?
                       # assoc.inverse_of is the belongs_to side of the has_many train we came in here on.
                       sub_hm, criteria = assoc.klass.find_existing(uniques, cols, starred, import_template, keepers, assoc.inverse_of, row, sub_next, all, trim_prefix)
-
                       # If still not found then create a new related object using this has_many collection
                       # (criteria.empty? ? nil : sub_next.new(criteria))
-                      sub_next = sub_hm || sub_next.new(criteria)
+                      if sub_hm
+                        sub_next = sub_hm
+                      elsif assoc.macro == :has_one
+                        bt_name = "#{assoc.inverse.name}="
+                        sub_next = assoc.klass.new(criteria)
+                        to_be_saved << [sub_next, sub_obj, bt_name, sub_next]
+                      else
+                        # Two other methods that are possible to check for here are :conditions and
+                        # :sanitized_conditions, which do not exist in Rails 4.0 and later.
+                        sub_next = if assoc.respond_to?(:require_association)
+                                     # With Rails < 4.0, sub_next could come in as an Array or a broken CollectionProxy
+                                     assoc.klass.new({ fk_from(assoc) => sub_obj.send(sub_obj.class.primary_key) }.merge(criteria))
+                                   else
+                                     sub_next.proxy_association.reflection.instance_variable_set(:@foreign_key, fk_from(assoc))
+                                     sub_next.new(criteria)
+                                   end
+                        to_be_saved << [sub_next]
+                      end
+                    end
+                    # Look for possible missing polymorphic detail
+                    # Maybe can test for this via assoc.through_reflection
+                    if assoc.is_a?(ActiveRecord::Reflection::ThroughReflection) &&
+                       (delegate = assoc.send(:delegate_reflection)&.active_record&.reflect_on_association(assoc.source_reflection_name)) &&
+                       delegate.options[:polymorphic]
+                      polymorphics << { parent: sub_next, child: sub_obj, type_col: delegate.foreign_type, id_col: delegate.foreign_key.to_s }
                     end
                     unless sub_next.nil?
                       # if sub_next.class.name == devise_class && # only for Devise users
@@ -336,7 +337,7 @@ module DutyFree
                       sub_objects[this_path] = sub_next if this_path.present?
                     end
                   end
-                  sub_obj = sub_next # if sub_next
+                  sub_obj = sub_next
                 end
                 next if sub_obj.nil?
 
@@ -363,15 +364,16 @@ module DutyFree
                 # else
                 #   puts "  #{sub_obj.class.name} doesn't respond to #{sym}"
               end
+              ::DutyFree::Extensions._save_pending(to_be_saved)
               # Try to save a final sub-object if one exists
-              sub_obj.save if sub_obj && this_path && !is_has_one && sub_obj.valid?
+              # sub_obj.save if sub_obj && this_path && !is_has_one && sub_obj.valid?
 
-              # Wire up has_one associations
-              has_ones.each do |hasone|
-                parent = sub_objects[hasone[0..-2].map(&:to_s).join(',')] || obj
-                hasone_object = sub_objects[hasone.map(&:to_s).join(',')]
-                parent.send("#{hasone[-1]}=", hasone_object) if parent.new_record? || hasone_object.valid?
-              end
+              # # Wire up has_one associations
+              # has_ones.each do |hasone|
+              #   parent = sub_objects[hasone[0..-2].map(&:to_s).join(',')] || obj
+              #   hasone_object = sub_objects[hasone.map(&:to_s).join(',')]
+              #   parent.send("#{hasone[-1]}=", hasone_object) if parent.new_record? || hasone_object.valid?
+              # end
 
               # Reinstate any missing polymorphic _type and _id values
               polymorphics.each do |poly|
@@ -425,11 +427,46 @@ module DutyFree
         ret
       end
 
+      def fk_from(assoc)
+        # Try first to trust whatever they've marked as being the foreign_key, and then look
+        # at the inverse's foreign key setting if available.  In all cases don't accept
+        # anything that's not backed with a real column in the table.
+        col_names = assoc.klass.column_names
+        if (fk_name = assoc.options[:foreign_key]) && col_names.include?(fk_name.to_s)
+          return fk_name
+        end
+        if (fk_name = assoc.inverse_of&.options&.fetch(:foreign_key) { nil }) &&
+           col_names.include?(fk_name.to_s)
+          return fk_name
+        end
+        if assoc.respond_to?(:foreign_key) && (fk_name = assoc.foreign_key) &&
+           col_names.include?(fk_name.to_s)
+          return fk_name
+        end
+        if (fk_name = assoc.inverse_of&.foreign_key) &&
+           col_names.include?(fk_name.to_s)
+          return fk_name
+        end
+        if (fk_name = assoc.inverse_of&.association_foreign_key) &&
+           col_names.include?(fk_name.to_s)
+          return fk_name
+        end
+        # Don't let this fool you -- we really are in search of the foreign key name here,
+        # and Rails 3.0 and older used some fairly interesting conventions, calling it instead
+        # the "primary_key_name"!
+        if assoc.respond_to?(:primary_key_name) && (fk_name = assoc.primary_key_name) &&
+           col_names.include?(fk_name.to_s)
+          return fk_name
+        end
+
+        puts "* Wow, had no luck at all finding a foreign key for #{assoc.inspect}"
+      end
+
       # For use with importing, based on the provided column list calculate all valid combinations
       # of unique columns.  If there is no valid combination, throws an error.
       # Returns an object found by this means.
       def find_existing(uniques, cols, starred, import_template, keepers, train_we_came_in_here_on,
-                        row = nil, klass_or_collection = nil, all = nil, trim_prefix = '')
+                        row = nil, klass_or_collection = nil, template_all = nil, trim_prefix = '')
         unless trim_prefix.blank?
           cols = cols.map { |c| c.start_with?(trim_prefix) ? c[trim_prefix.length..-1] : nil }
           starred = starred.each_with_object([]) do |v, s|
@@ -451,11 +488,16 @@ module DutyFree
 
         vus = ((@valid_uniques ||= {})[col_list] ||= {}) # Fancy memoisation
 
+        # First, get an overall list of AVAILABLE COLUMNS before considering tricky foreign key stuff.
+        # ============================================================================================
+        # Generate a list of column names matched up with their zero-ordinal column number mapping for
+        # all columns from the incoming import data.
         if (is_new_vus = vus.empty?)
-          #   # Let's do general attributes before the tricky foreign key stuff
-          # Find all unique combinations that are available based on incoming columns, and
-          # pair them up with column number mappings.
-          template_column_objects = ::DutyFree::Extensions._recurse_def(self, all || import_template[:all], import_template).first
+          template_column_objects = ::DutyFree::Extensions._recurse_def(
+            self,
+            template_all || import_template[:all],
+            import_template
+          ).first
           available = if trim_prefix.blank?
                         template_column_objects.select { |col| col.pre_prefix.blank? && col.prefix.blank? }
                       else
@@ -465,15 +507,18 @@ module DutyFree
                           trim_prefix_snake == "#{this_prefix}_"
                         end
                       end.map { |avail| avail.name.to_s.titleize }
-          all_vus = defined_uniques(uniques, cols, nil, starred, trim_prefix)
-          #   k, v = all_vus.first
-          #   k.each_with_index do |col, idx|
-          #     if available.include?(col) # || available_bts.include?(col)
-          #       vus[col] ||= v[idx]
-          #     end
-          #     # if available_bts.include?(k)
         end
 
+        # Process FOREIGN KEY stuff by going through each belongs_to in this model.
+        # =========================================================================
+        # This list of all valid uniques will help to filter which foreign keys are kept, and also
+        # get further filtered later to arrive upon a final set of valid uniques.  (Often but not
+        # necessarily a specific valid unique as perhaps with a list of users you want to update some
+        # folks based on having their email as a unique identifier, and other folks by having a
+        # combination of their name and street address as unique, and use both of those possible
+        # unique variations to update phone numbers, and do that all as a part of one import.)
+        all_vus = defined_uniques(uniques, cols, col_list, starred, trim_prefix)
+
         # %%% Ultimately may consider making this recursive
         reflect_on_all_associations.each do |sn_bt|
           next unless sn_bt.belongs_to? && (!train_we_came_in_here_on || sn_bt != train_we_came_in_here_on)
@@ -488,7 +533,7 @@ module DutyFree
 
             available_bts << bt_col
             fk_id = if row
-                      # Max ID so if there are multiple, only the most recent one is picked.
+                      # Max ID so if there are multiple matches, only the most recent one is picked.
                       # %%% Need to stack these up in case there are multiple
                       # (like first_name, last_name on a referenced employee)
                       sn_bt.klass.where(keepers[idx].name => row[idx]).limit(1).pluck(MAX_ID).first
@@ -519,15 +564,25 @@ module DutyFree
               bt_col_indexes << idx
               bt_criteria_all_nil = false
             end
+            # If we're processing a row then this list of foreign key column name entries, named such as
+            # "order_id" or "product_id" instead of column-specific stuff like "Order Date" and "Product Name",
+            # is kept until the last and then gets merged on top of the other criteria before being returned.
             bt_criteria[(fk_name = sn_bt.foreign_key)] = fk_id
 
-            # Add to our criteria if this belongs_to is required
+            # Check to see if belongs_tos are generally required on this specific table
             bt_req_by_default = sn_bt.klass.respond_to?(:belongs_to_required_by_default) &&
                                 sn_bt.klass.belongs_to_required_by_default
 
-            # The first check, "!all_vus.keys.first.exists { |k| k.start_with?(bt_prefix) }"
+            # Add to our CRITERIA just the belongs_to things that check out.
+            # ==============================================================
+            # The first check, "all_vus.keys.first.none? { |k| k.start_with?(bt_prefix) }"
             # is to see if one of the columns we're working with from the unique that we've chosen
             # comes from the table referenced by this belongs_to (sn_bt).
+            #
+            # The second check on the :optional option and bt_req_by_default comes directly from
+            # how Rails 5 and later checks to see if a specific belongs_to is marked optional
+            # (or required), and without having that indication will fall back on checking the model
+            # itself to see if it requires belongs_tos by default.
             next if all_vus.keys.first.none? { |k| k.start_with?(bt_prefix) } &&
                     (sn_bt.options[:optional] || !bt_req_by_default)
 
@@ -536,6 +591,8 @@ module DutyFree
           end
         end
 
+        # Now circle back find a final list of VALID UNIQUES by re-assessing the list of all valid uniques
+        # in relation to the available belongs_tos found in the last foreign key step.
         if is_new_vus
           available += available_bts
           all_vus.each do |k, v|
@@ -585,21 +642,21 @@ module DutyFree
           end
         end
 
-        # Short-circuiting this to only get back the valid_uniques?
-        # unless uniq_lookups == criteria
-        #   puts "Compare #{uniq_lookups.inspect}"
-        #   puts "Compare #{criteria.inspect}"
-        # end
         return uniq_lookups.merge(criteria) if only_valid_uniques
 
         # If there's nothing to match upon then we're out
         return [nil, {}] if new_criteria_all_nil
 
-        # With this criteria, find any matching has_many row we can so we can update it
-        # First try looking it up through ActiveRecord
+        # With this criteria, find any matching has_many row we can so we can update it.
+        # First try directly looking it up through ActiveRecord.
         found_object = klass_or_collection.find_by(criteria)
         # If not successful, such as when fields are exposed via helper methods instead of being
-        # real columns in the database tables, try this more intensive routine.
+        # real columns in the database tables, try this more intensive approach.  This is useful
+        # if you had full name kind of data coming in on a spreadsheeet, but in the destination
+        # table it's broken out to first_name, middle_name, surname.  By writing both full_name
+        # and full_name= methods, the importer can check to see if this entry is already there,
+        # and put a new row in if not, having one incoming name break out to populate three
+        # destination columns.
         unless found_object || klass_or_collection.is_a?(Array)
           found_object = klass_or_collection.find do |obj|
             is_good = true
@@ -612,6 +669,9 @@ module DutyFree
             is_good
           end
         end
+        # Standard criteria as well as foreign key column name detail with exact foreign keys
+        # that match up to a primary key so that if needed a new related object can be built,
+        # complete with all its association detail.
         [found_object, criteria.merge(bt_criteria)]
       end
 
@@ -657,6 +717,32 @@ module DutyFree
       end
     end # module ClassMethods
 
+    # Called before building any object linked through a has_one or has_many so that foreign key IDs
+    # can be added properly to those new objects.  Finally at the end also called to save everything.
+    def self._save_pending(to_be_saved)
+      while (tbs = to_be_saved.pop)
+        ais = (tbs.first.class.respond_to?(:around_import_save) && tbs.first.class.method(:around_import_save)) ||
+              (respond_to?(:around_import_save) && method(:around_import_save))
+        if ais
+          # Send them the sub_obj even if it might be invalid so they can choose
+          # to make it valid if they wish.
+          ais.call(tbs.first) do |modded_obj = nil|
+            modded_obj = (modded_obj || tbs.first)
+            modded_obj.save if modded_obj&.valid?
+          end
+        elsif tbs.first.valid?
+          tbs.first.save
+        else
+          puts "* Unable to save #{tbs.first.inspect}"
+        end
+        # puts "Save #{tbs.first.class.name} #{tbs.first&.id} #{!tbs.first.new_record?}"
+        unless tbs[1].nil? || tbs.first.new_record?
+          # puts "Calling #{tbs[1].class.name} #{tbs[1]&.id} .#{tbs[2]} #{tbs[3].class.name} #{tbs[3]&.id}"
+          tbs[1].send(tbs[2], tbs[3])
+        end
+      end
+    end
+
     # The snake-cased column alias names used in the query to export data
     def self._template_columns(klass, import_template = nil)
       template_detail_columns = klass.instance_variable_get(:@template_detail_columns)
@@ -674,16 +760,21 @@ module DutyFree
 
     # Recurse and return two arrays -- one with all columns in sequence, and one a hierarchy of
     # nested hashes to be used with ActiveRecord's .joins() to facilitate export.
-    def self._recurse_def(klass, array, import_template, assocs = [], joins = [], pre_prefix = '', prefix = '')
+    def self._recurse_def(klass, array, import_template, order_by = [], assocs = [], joins = [], pre_prefix = '', prefix = '')
+      prefixes = ::DutyFree::Util._prefix_join([pre_prefix, prefix])
       # Confirm we can actually navigate through this association
       prefix_assoc = (assocs.last&.klass || klass).reflect_on_association(prefix) if prefix.present?
-      assocs = assocs.dup << prefix_assoc unless prefix_assoc.nil?
-      prefixes = ::DutyFree::Util._prefix_join([pre_prefix, prefix])
+      if prefix_assoc
+        assocs = assocs.dup << prefix_assoc
+        if prefix_assoc.macro == :has_many && (pk = prefix_assoc.active_record.primary_key)
+          order_by << ["#{prefixes.tr('.', '_')}_", pk]
+        end
+      end
       array = array.inject([]) do |s, col|
         s + if col.is_a?(Hash)
               col.inject([]) do |s2, v|
                 joins << { v.first.to_sym => (joins_array = []) }
-                s2 + _recurse_def(klass, (v.last.is_a?(Array) ? v.last : [v.last]), import_template, assocs, joins_array, prefixes, v.first.to_sym).first
+                s2 + _recurse_def(klass, (v.last.is_a?(Array) ? v.last : [v.last]), import_template, order_by, assocs, joins_array, prefixes, v.first.to_sym).first
               end
             elsif col.nil?
               if assocs.empty?
@@ -692,7 +783,7 @@ module DutyFree
                 # Bring in from another class
                 joins << { prefix => (joins_array = []) }
                 # %%% Also bring in uniques and requireds
-                _recurse_def(klass, assocs.last.klass::IMPORT_TEMPLATE[:all], import_template, assocs, joins_array, prefixes).first
+                _recurse_def(klass, assocs.last.klass::IMPORT_TEMPLATE[:all], import_template, order_by, assocs, joins_array, prefixes).first
               end
             else
               [::DutyFree::Column.new(col, pre_prefix, prefix, assocs, klass, import_template[:as])]

data/lib/duty_free/suggest_template.rb

@@ -65,7 +65,7 @@ module DutyFree
         end
         next if is_through || is_habtm || (!is_belongs_to && !do_has_many) || errored_assocs.include?(assoc)
 
-        if is_belongs_to && assoc.polymorphic? # Polymorphic belongs_to?
+        if is_belongs_to && assoc.options[:polymorphic] # Polymorphic belongs_to?
           # Load all models
           # %%% Note that this works in Rails 5.x, but may not work in Rails 6.0 and later, which uses the Zeitwerk loader by default:
           Rails.configuration.eager_load_namespaces.select { |ns| ns < Rails::Application }.each(&:eager_load!)
@@ -89,7 +89,7 @@ module DutyFree
           end
         else
           # Is it a polymorphic has_many, which is defined using as: :somethingable ?
-          is_polymorphic_hm = assoc.inverse_of&.polymorphic?
+          is_polymorphic_hm = assoc.inverse_of&.options&.fetch(:polymorphic) { nil }
           begin
             # Standard has_one, or has_many, and belongs_to uses assoc.klass.
             # Also polymorphic belongs_to uses assoc.klass.
@@ -105,9 +105,9 @@ module DutyFree
               [[[fk], assoc.active_record], assoc_klass]
             else # has_many or has_one
               inverse_foreign_keys = is_polymorphic_hm ? [assoc.type, assoc.foreign_key] : [assoc.inverse_of&.foreign_key&.to_s]
-              puts "* Missing inverse foreign key for #{assoc.inspect}" if inverse_foreign_keys.first.nil?
               missing_key_columns = inverse_foreign_keys - assoc_klass.columns.map(&:name)
               if missing_key_columns.empty?
+                puts "* Missing inverse foreign key for #{this_klass.name} #{belongs_to_or_has_many} :#{assoc.name}" if inverse_foreign_keys.first.nil?
                 # puts "Has columns #{inverse_foreign_keys.inspect}"
                 [[inverse_foreign_keys, assoc_klass], assoc_klass]
               else
@@ -185,7 +185,7 @@ module DutyFree
         next unless bt_assoc.belongs_to? && !errored_assocs.include?(bt_assoc)
 
         begin
-          s << bt_assoc if !bt_assoc.polymorphic? && bt_assoc.klass == to_klass
+          s << bt_assoc if !bt_assoc.options[:polymorphic] && bt_assoc.klass == to_klass
         rescue NameError
           errored_assocs << bt_assoc
           puts "* In the #{bt_assoc.active_record.name} model  \"belongs_to :#{bt_assoc.name}\"  could not find a model named #{bt_assoc.class_name}."

data/lib/duty_free/util.rb

@@ -45,7 +45,7 @@ module DutyFree
           # The right side here at the top is the very last table, and anywhere else down the tree it is
           # the later "JOIN" table of this pair.  (The table that comes after all the rest of the JOINs
           # from the left side.)
-          names << piece.right.name
+          names << (piece.right.table_alias || piece.right.name)
         else # "Normal" setup, fed from a JoinSource which has an array of JOINs
           # The left side is the "JOIN" table
           names += _recurse_arel(piece.left)
@@ -53,14 +53,14 @@ module DutyFree
         end
         # rubocop:enable Style/IdenticalConditionalBranches
       elsif piece.is_a?(Arel::Table) # Table
-        names << piece.name
+        names << (piece.table_alias || piece.name)
       elsif piece.is_a?(Arel::Nodes::TableAlias) # Alias
         # Can get the real table name from:  self._recurse_arel(piece.left)
         names << piece.right.to_s # This is simply a string; the alias name itself
       elsif piece.is_a?(Arel::Nodes::JoinSource) # Leaving this until the end because AR < 3.2 doesn't know at all about JoinSource!
         # The left side is the "FROM" table
         # names += _recurse_arel(piece.left)
-        names << piece.left.name
+        names << (piece.left.table_alias || piece.left.name)
         # The right side is an array of all JOINs
         names += piece.right.inject([]) { |s, v| s + _recurse_arel(v) }
       end

data/lib/duty_free/version_number.rb

@@ -5,7 +5,7 @@ module DutyFree
   module VERSION
     MAJOR = 1
     MINOR = 0
-    TINY = 5
+    TINY = 6
 
     # PRE is nil unless it's a pre-release (beta, RC, etc.)
     PRE = nil

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: duty_free
 version: !ruby/object:Gem::Version
-  version: 1.0.5
+  version: 1.0.6
 platform: ruby
 authors:
 - Lorin Thwaits
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-20 00:00:00.000000000 Z
+date: 2020-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord