duty_free 1.0.1 → 1.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 87d07568c8c7a1692981435e098bd8426d17f30cd733ffff01a7b5a4e4fee4a9
-  data.tar.gz: 1d2ef595a271f40e5489cac67e576b4f85338d43d46bff26c6e5a6d706766a00
+  metadata.gz: f9ac922dd807fc18539ff130c1ccf63412e837acbe4eba0127423235c11d0f1e
+  data.tar.gz: e5c27fe4a58718dcbbd4c5ebc1aed2dbb5e7e4b386c977e0ec31269cfad08d48
 SHA512:
-  metadata.gz: 4bdd6c53f65cb096de1c2da724a735ecee7ceaed464627f69adbb816b3afa20c85a202cce201a7ee5d35e52ae31663c7f8a9c02d47bc11ea7ead86cebfb0972f
-  data.tar.gz: 4664e2b6e724f3bba469ef1907b82fe25d90761251ff5cabb204bd75e1a64b522ba3a438438a76e5d58e80952870334abb8194d9d0199b6e2addb2143120b5fa
+  metadata.gz: 54811a68bd34fd5bb881fa19006a7297a8697a3a1575b7060ba7d82e6fcd6d36f545e1d5cfcfc51c756a5b87880e445c415924d2f54aa39471f0b884f0b4c6c4
+  data.tar.gz: 32ccb88e429e64bccb59c7741955b99537c661d6416d6f8c352cb384df728081d5b54079cb2dbd904203fbcf1c60abb1a4a7c70bee9c2764fa1622eaaeb6843f

data/lib/duty_free.rb

@@ -1,5 +1,47 @@
 # frozen_string_literal: true
 
+require 'active_record/version'
+
+# ActiveRecord before 4.0 didn't have #version
+unless ActiveRecord.respond_to?(:version)
+  module ActiveRecord
+    def self.version
+      ::Gem::Version.new(ActiveRecord::VERSION::STRING)
+    end
+  end
+end
+
+# In ActiveSupport older than 5.0, the duplicable? test tries to new up a BigDecimal,
+# and Ruby 2.6 and later deprecates #new.  This removes the warning from BigDecimal.
+require 'bigdecimal'
+if ActiveRecord.version < ::Gem::Version.new('5.0') &&
+   ::Gem::Version.new(RUBY_VERSION) >= ::Gem::Version.new('2.6')
+  def BigDecimal.new(*args, **kwargs)
+    BigDecimal(*args, **kwargs)
+  end
+end
+
+# Allow Rails 4.0 and 4.1 to work with newer Ruby (>= 2.4) by avoiding a "stack level too deep" error
+# when ActiveSupport tries to smarten up Numeric by messing with Fixnum and Bignum at the end of:
+# activesupport-4.0.13/lib/active_support/core_ext/numeric/conversions.rb
+if ActiveRecord.version < ::Gem::Version.new('4.2') &&
+   ActiveRecord.version > ::Gem::Version.new('3.2') &&
+   Object.const_defined?('Integer') && Integer.superclass.name == 'Numeric'
+  class OurFixnum < Integer; end
+  Numeric.const_set('Fixnum', OurFixnum)
+  class OurBignum < Integer; end
+  Numeric.const_set('Bignum', OurBignum)
+end
+
+# Allow Rails < 3.2 to run with newer versions of Psych gem
+if BigDecimal.respond_to?(:yaml_tag) && !BigDecimal.respond_to?(:yaml_as)
+  class BigDecimal
+    class <<self
+      alias yaml_as yaml_tag
+    end
+  end
+end
+
 require 'active_record'
 
 require 'duty_free/config'
@@ -62,7 +104,146 @@ module DutyFree
   end
 end
 
+# Major compatibility fixes for ActiveRecord < 4.2
+# ================================================
 ActiveSupport.on_load(:active_record) do
+  # Rails < 4.0 cannot do #find_by, or do #pluck on multiple columns, so here are the patches:
+  if ActiveRecord.version < ::Gem::Version.new('4.0')
+    module ActiveRecord
+      module Calculations # Normally find_by is in FinderMethods, which older AR doesn't have
+        def find_by(*args)
+          where(*args).limit(1).to_a.first
+        end
+
+        def pluck(*column_names)
+          column_names.map! do |column_name|
+            if column_name.is_a?(Symbol) && self.column_names.include?(column_name.to_s)
+              "#{connection.quote_table_name(table_name)}.#{connection.quote_column_name(column_name)}"
+            else
+              column_name
+            end
+          end
+
+          # Same as:  if has_include?(column_names.first)
+          if eager_loading? || (includes_values.present? && (column_names.first || references_eager_loaded_tables?))
+            construct_relation_for_association_calculations.pluck(*column_names)
+          else
+            relation = clone # spawn
+            relation.select_values = column_names
+            result = if klass.connection.class.name.end_with?('::PostgreSQLAdapter')
+                       rslt = klass.connection.execute(relation.arel.to_sql)
+                       rslt.type_map =
+                         @type_map ||= proc do
+                           # This aliasing avoids the warning:
+                           # "no type cast defined for type "numeric" with oid 1700. Please cast this type
+                           # explicitly to TEXT to be safe for future changes."
+                           PG::BasicTypeRegistry.alias_type(0, 'numeric', 'text')
+                           PG::BasicTypeMapForResults.new(klass.connection.raw_connection)
+                         end.call
+                       rslt.to_a
+                     elsif respond_to?(:bind_values)
+                       klass.connection.select_all(relation.arel, nil, bind_values)
+                     else
+                       klass.connection.select_all(relation.arel.to_sql, nil)
+                     end
+            if result.empty?
+              []
+            else
+              columns = result.first.keys.map do |key|
+                # rubocop:disable Style/SingleLineMethods Naming/MethodParameterName
+                klass.columns_hash.fetch(key) do
+                  Class.new { def type_cast(v); v; end }.new
+                end
+                # rubocop:enable Style/SingleLineMethods Naming/MethodParameterName
+              end
+
+              result = result.map do |attributes|
+                values = klass.initialize_attributes(attributes).values
+
+                columns.zip(values).map do |column, value|
+                  column.type_cast(value)
+                end
+              end
+              columns.one? ? result.map!(&:first) : result
+            end
+          end
+        end
+      end
+
+      unless Base.is_a?(Calculations)
+        class Base
+          class << self
+            delegate :pluck, :find_by, to: :scoped
+          end
+        end
+      end
+
+      # ActiveRecord < 3.2 doesn't have initialize_attributes, used by .pluck()
+      unless AttributeMethods.const_defined?('Serialization')
+        class Base
+          class << self
+            def initialize_attributes(attributes, options = {}) #:nodoc:
+              serialized = (options.delete(:serialized) { true }) ? :serialized : :unserialized
+              # super(attributes, options)
+
+              serialized_attributes.each do |key, coder|
+                attributes[key] = Attribute.new(coder, attributes[key], serialized) if attributes.key?(key)
+              end
+
+              attributes
+            end
+          end
+        end
+      end
+
+      # This only gets added for ActiveRecord < 3.2
+      module Reflection
+        unless AssociationReflection.instance_methods.include?(:foreign_key)
+          class AssociationReflection < MacroReflection
+            alias foreign_key association_foreign_key
+          end
+        end
+      end
+    end
+  end
+
+  # Rails < 4.2 is not innately compatible with Ruby 2.4 and later, and comes up with:
+  # "TypeError: Cannot visit Integer" unless we patch like this:
+  unless ::Gem::Version.new(RUBY_VERSION) < ::Gem::Version.new('2.4')
+    unless Arel::Visitors::DepthFirst.private_instance_methods.include?(:visit_Integer)
+      module Arel
+        module Visitors
+          class DepthFirst < Visitor
+            alias visit_Integer terminal
+          end
+
+          class Dot < Visitor
+            alias visit_Integer visit_String
+          end
+
+          class ToSql < Visitor
+          private
+
+            # ActiveRecord before v3.2 uses Arel < 3.x, which does not have Arel#literal.
+            unless private_instance_methods.include?(:literal)
+              def literal(obj)
+                obj
+              end
+            end
+            alias visit_Integer literal
+          end
+        end
+      end
+    end
+  end
+
+  if ActiveRecord.version < ::Gem::Version.new('5.0')
+    # Avoid pg gem deprecation warning:  "You should use PG::Connection, PG::Result, and PG::Error instead"
+    PGconn = PG::Connection
+    PGresult = PG::Result
+    PGError = PG::Error
+  end
+
   include ::DutyFree::Extensions
 end
 

data/lib/duty_free/column.rb

@@ -28,25 +28,23 @@ module DutyFree
       # sql_col = ::DutyFree::Util._prefix_join([table_name.compact.join('_'), name])
 
       # Foolproof way, using the AREL mapping:
-      sql_col = ::DutyFree::Util._prefix_join([mapping["#{pre_prefix.tr('.', '_')}_#{prefix}_"], name])
+      this_pre_prefix = pre_prefix.tr('.', '_')
+      this_pre_prefix << '_' unless pre_prefix.blank?
+      sql_col = ::DutyFree::Util._prefix_join([mapping["#{this_pre_prefix}#{prefix}_"], name])
       sym = to_sym.to_s
       sql_col == sym ? sql_col : "#{sql_col} AS #{sym}"
     end
 
     def titleize
-      @titleized ||= sym_string.titleize
+      @titleized ||= to_sym.titleize
     end
 
-    delegate :to_sym, to: :sym_string
-
     def path
       @path ||= ::DutyFree::Util._prefix_join([pre_prefix, prefix]).split('.').map(&:to_sym)
     end
 
-  private
-
     # The snake-cased column name to be used for building the full list of template_columns
-    def sym_string
+    def to_sym
       @sym_string ||= ::DutyFree::Util._prefix_join(
         [pre_prefix, prefix, ::DutyFree::Util._clean_name(name, import_template_as)],
         '_'

data/lib/duty_free/extensions.rb

@@ -14,11 +14,13 @@ module DutyFree
 
     # :nodoc:
     module ClassMethods
+      MAX_ID = Arel.sql('MAX(id)')
       # def self.extended(model)
       # end
 
       # Export at least column header, and optionally include all existing data as well
-      def df_export(is_with_data = true, import_template = nil)
+      def df_export(is_with_data = true, import_template = nil, use_inner_joins = false)
+        use_inner_joins = true unless respond_to?(:left_joins)
         # In case they are only supplying the columns hash
         if is_with_data.is_a?(Hash) && !import_template
           import_template = is_with_data
@@ -46,20 +48,31 @@ module DutyFree
           end
           (is_required ? '* ' : '') + col
         end
-        rows = Array(rows)
+        rows = [rows]
 
         if is_with_data
+          order_by = []
+          order_by << ['_', primary_key] if primary_key
           # Automatically create a JOINs strategy and select list to get back all related rows
-          template_cols, template_joins = ::DutyFree::Extensions._recurse_def(self, import_template[:all], import_template)
-          relation = left_joins(template_joins)
+          template_cols, template_joins = ::DutyFree::Extensions._recurse_def(self, import_template[:all], import_template, order_by)
+          relation = use_inner_joins ? joins(template_joins) : left_joins(template_joins)
 
           # So we can properly create the SELECT list, create a mapping between our
           # column alias prefixes and the aliases AREL creates.
-          # Warning: Delegating ast to arel is deprecated and will be removed in Rails 6.0
-          arel_alias_names = ::DutyFree::Util._recurse_arel(relation.ast.cores.first.source)
-          our_names = ::DutyFree::Util._recurse_arel(template_joins)
+          core = relation.arel.ast.cores.first
+          # Accommodate AR < 3.2
+          arel_alias_names = if core.froms.is_a?(Arel::Table)
+                               # All recent versions of AR have #source which brings up an Arel::Nodes::JoinSource
+                               ::DutyFree::Util._recurse_arel(core.source)
+                             else
+                               # With AR < 3.2, "froms" brings up the top node, an Arel::Nodes::InnerJoin
+                               ::DutyFree::Util._recurse_arel(core.froms)
+                             end
+          our_names = ['_'] + ::DutyFree::Util._recurse_arel(template_joins)
           mapping = our_names.zip(arel_alias_names).to_h
-
+          relation = (order_by.empty? ? relation : relation.order(order_by.map { |o| "#{mapping[o.first]}.#{o.last}" }))
+          # puts mapping.inspect
+          # puts relation.dup.select(template_cols.map { |x| x.to_s(mapping) }).to_sql
           relation.select(template_cols.map { |x| x.to_s(mapping) }).each do |result|
             rows << ::DutyFree::Extensions._template_columns(self, import_template).map do |col|
               value = result.send(col)
@@ -79,15 +92,15 @@ module DutyFree
 
       # With an array of incoming data, the first row having column names, perform the import
       def df_import(data, import_template = nil)
-        self.instance_variable_set(:@defined_uniques, nil)
-        self.instance_variable_set(:@valid_uniques, nil)
+        instance_variable_set(:@defined_uniques, nil)
+        instance_variable_set(:@valid_uniques, nil)
 
         import_template ||= if constants.include?(:IMPORT_TEMPLATE)
                               self::IMPORT_TEMPLATE
                             else
                               suggest_template(0, false, false)
                             end
-        puts "Chose #{import_template}"
+        # puts "Chose #{import_template}"
         inserts = []
         updates = []
         counts = Hash.new { |h, k| h[k] = [] }
@@ -105,6 +118,7 @@ module DutyFree
         devise_class = ''
         ret = nil
 
+        # Multi-tenancy gem Apartment can be used if there are separate schemas per tenant
         reference_models = if Object.const_defined?('Apartment')
                              Apartment.excluded_models
                            else
@@ -121,10 +135,13 @@ module DutyFree
 
         # Did they give us a filename?
         if data.is_a?(String)
-          data = if data.length <= 4096 && data.split('\n').length == 1
+          # Filenames with full paths can not be longer than 4096 characters, and can not
+          # include newline characters
+          data = if data.length <= 4096 && !data.index('\n')
                    File.open(data)
                  else
-                   # Hope that other multi-line strings might be CSV data
+                   # Any multi-line string is likely CSV data
+                   # %%% Test to see if TAB characters are present on the first line, instead of commas
                    CSV.new(data)
                  end
         end
@@ -143,10 +160,16 @@ module DutyFree
         # Will show as just one transaction when using auditing solutions such as PaperTrail
         ActiveRecord::Base.transaction do
           # Check to see if they want to do anything before the whole import
-          if before_import ||= (import_template[:before_import]) # || some generic before_import)
-            before_import.call(data)
+          # First if defined in the import_template, then if there is a method in the class,
+          # and finally (not yet implemented) a generic global before_import
+          my_before_import = import_template[:before_import]
+          my_before_import ||= respond_to?(:before_import) && method(:before_import)
+          # my_before_import ||= some generic my_before_import
+          if my_before_import
+            last_arg_idx = my_before_import.parameters.length - 1
+            arguments = [data, import_template][0..last_arg_idx]
+            data = ret if (ret = my_before_import.call(*arguments)).is_a?(Enumerable)
           end
-          col_list = nil
           data.each_with_index do |row, row_num|
             row_errors = {}
             if is_first # Anticipate that first row has column names
@@ -175,11 +198,8 @@ module DutyFree
                   col.strip!
                 end
               end
-              # %%% Will the uniques saved into @defined_uniques here just get redefined later
-              # after the next line, the map! with clean to change out the alias names?  So we can't yet set
-              # col_list?
+              cols.map! { |col| ::DutyFree::Util._clean_name(col, import_template[:as]) }
               defined_uniques(uniques, cols, cols.join('|'), starred)
-              cols.map! { |col| ::DutyFree::Util._clean_name(col, import_template[:as]) } # %%%
               # Make sure that at least half of them match what we know as being good column names
               template_column_objects = ::DutyFree::Extensions._recurse_def(self, import_template[:all], import_template).first
               cols.each_with_index do |col, idx|
@@ -187,9 +207,7 @@ module DutyFree
                 keepers[idx] = template_column_objects.find { |col_obj| col_obj.titleize == col }
                 # puts "Could not find a match for column #{idx + 1}, #{col}" if keepers[idx].nil?
               end
-              if keepers.length < (cols.length / 2) - 1
-                raise ::DutyFree::LessThanHalfAreMatchingColumnsError, I18n.t('import.altered_import_template_coumns')
-              end
+              raise ::DutyFree::LessThanHalfAreMatchingColumnsError, I18n.t('import.altered_import_template_coumns') if keepers.length < (cols.length / 2) - 1
 
               # Returns just the first valid unique lookup set if there are multiple
               valid_unique = find_existing(uniques, cols, starred, import_template, keepers, false)
@@ -201,24 +219,22 @@ module DutyFree
               is_first = false
             else # Normal row of data
               is_insert = false
-              is_do_save = true
               existing_unique = valid_unique.inject([]) do |s, v|
                 s << if v.last.is_a?(Array)
-                       v.last[0].where(v.last[1] => row[v.last[2]]).limit(1).pluck('MAX(id)').first.to_s
+                       v.last[0].where(v.last[1] => row[v.last[2]]).limit(1).pluck(MAX_ID).first.to_s
                      else
                        row[v.last].to_s
                      end
               end
+              to_be_saved = []
               # Check to see if they want to preprocess anything
-              if @before_process ||= import_template[:before_process]
-                existing_unique = @before_process.call(valid_unique, existing_unique)
+              existing_unique = @before_process.call(valid_unique, existing_unique) if @before_process ||= import_template[:before_process]
+              if (criteria = existing[existing_unique])
+                obj = find(criteria)
+              else
+                is_insert = true
+                to_be_saved << [obj = new]
               end
-              obj = if existing.include?(existing_unique)
-                      find(existing[existing_unique])
-                    else
-                      is_insert = true
-                      new
-                    end
               sub_obj = nil
               is_has_one = false
               has_ones = []
@@ -226,31 +242,15 @@ module DutyFree
               sub_objects = {}
               this_path = nil
               keepers.each do |key, v|
-                klass = nil
                 next if v.nil?
 
-                # Not the same as the last path?
-                if this_path && v.path != this_path.split(',').map(&:to_sym) && !is_has_one
-                  # puts sub_obj.class.name
-                  if respond_to?(:around_import_save)
-                    # Send them the sub_obj even if it might be invalid so they can choose
-                    # to make it valid if they wish.
-                    # binding.pry
-                    around_import_save(sub_obj) do |modded_obj = nil|
-                      modded_obj = (modded_obj || sub_obj)
-                      modded_obj.save if sub_obj&.valid?
-                    end
-                  elsif sub_obj&.valid?
-                    # binding.pry if sub_obj.is_a?(Employee) && sub_obj.first_name == 'Andrew'
-                    sub_obj.save
-                  end
-                end
                 sub_obj = obj
                 this_path = +''
+                # puts "p: #{v.path}"
                 v.path.each_with_index do |path_part, idx|
                   this_path << (this_path.blank? ? path_part.to_s : ",#{path_part}")
                   unless (sub_next = sub_objects[this_path])
-                    # Check if we're hitting platform data / a lookup thing
+                    # Check if we're hitting reference data / a lookup thing
                     assoc = v.prefix_assocs[idx]
                     # belongs_to some lookup (reference) data
                     if assoc && reference_models.include?(assoc.class_name)
@@ -262,74 +262,68 @@ module DutyFree
                         lookup_match = (lookup_match.length == 1 ? lookup_match.first : nil)
                       end
                       sub_obj.send("#{path_part}=", lookup_match) unless lookup_match.nil?
-                      # Reference data from the platform level means we stop here
+                      # Reference data from the public level means we stop here
                       sub_obj = nil
                       break
                     end
-                    # This works for belongs_to or has_one.  has_many gets sorted below.
                     # Get existing related object, or create a new one
-                    if (sub_next = sub_obj.send(path_part)).nil?
-                      is_has_one = assoc.is_a?(ActiveRecord::Reflection::HasOneReflection)
+                    # This first part works for belongs_to.  has_many and has_one get sorted below.
+                    if (sub_next = sub_obj.send(path_part)).nil? && assoc.belongs_to?
                       klass = Object.const_get(assoc&.class_name)
-                      sub_next = if is_has_one
-                                   has_ones << v.path
-                                   klass.new
-                                 else
-                                   # Try to find a unique item if one is referenced
-                                   sub_bt = nil
-                                   begin
-                                     # Goofs up if trim_prefix isn't the same name as the class, or if it's
-                                     # a self-join?  (like when trim_prefix == 'Reports To')
-                                     # %%% Need to test this more when the self-join is more than one hop away,
-                                     # such as importing orders and having employees come along :)
-                                     # if sub_obj.class == klass
-                                     #   trim_prefix = ''
-                                     #   # binding.pry
-                                     # end
-                                     # %%% Maybe instead of passing in "klass" we can give the belongs_to association and build through that instead,
-                                     # allowing us to nix the klass.new(criteria) line below.
-                                     trim_prefix = v.titleize[0..-(v.name.length + 2)]
-                                     trim_prefix << ' ' unless trim_prefix.blank?
-                                     if klass == sub_obj.class # Self-referencing thing pointing to us?
-                                       # %%% This should be more general than just for self-referencing things.
-                                       sub_cols = cols.map { |c| c.start_with?(trim_prefix) ? c[trim_prefix.length..-1] : nil }
-                                       sub_bt, criteria = klass.find_existing(uniques, sub_cols, starred, import_template, keepers, assoc, row, klass, all, trim_prefix)
-                                     else
-                                       sub_bt, criteria = klass.find_existing(uniques, cols, starred, import_template, keepers, nil, row, klass, all, trim_prefix)
-                                     end
-                                   rescue ::DutyFree::NoUniqueColumnError
-                                     sub_unique = nil
-                                   end
-                                   # binding.pry if sub_obj.is_a?(Employee) && sub_obj.first_name == 'Nancy' &&
-                                   #   sub_bt.is_a?(Employee)
-                                   # %%% Can criteria really ever be nil anymore?
-                                   sub_bt ||= klass.new(criteria || {})
-                                   sub_obj.send("#{path_part}=", sub_bt)
-                                   sub_bt # unless klass == sub_obj.class # Don't go further if it's self-referencing
-                                 end
-                    end
-                    # Look for possible missing polymorphic detail
-                    if assoc.is_a?(ActiveRecord::Reflection::ThroughReflection) &&
-                       (delegate = assoc.send(:delegate_reflection)&.active_record&.reflect_on_association(assoc.source_reflection_name)) &&
-                       delegate.options[:polymorphic]
-                      polymorphics << { parent: sub_next, child: sub_obj, type_col: delegate.foreign_type, id_col: delegate.foreign_key.to_s }
-                    end
-                    # From a has_many?
-                    if sub_next.is_a?(ActiveRecord::Associations::CollectionProxy)
+                      # Try to find a unique item if one is referenced
+                      sub_next = nil
+                      begin
+                        trim_prefix = v.titleize[0..-(v.name.length + 2)]
+                        trim_prefix << ' ' unless trim_prefix.blank?
+                        sub_next, criteria = klass.find_existing(uniques, cols, starred, import_template, keepers, nil, row, klass, all, trim_prefix)
+                      rescue ::DutyFree::NoUniqueColumnError
+                      end
+                      # puts "#{v.path} #{criteria.inspect}"
+                      bt_name = "#{path_part}="
+                      unless sub_next || (klass == sub_obj.class && criteria.empty?)
+                        sub_next = klass.new(criteria || {})
+                        to_be_saved << [sub_next, sub_obj, bt_name, sub_next]
+                      end
+                      sub_obj.send(bt_name, sub_next)
+                    # From a has_many or has_one?
+                    # Rails 4.0 and later can do:  sub_next.is_a?(ActiveRecord::Associations::CollectionProxy)
+                    elsif [:has_many, :has_one].include?(assoc.macro) && !assoc.options[:through]
+                      ::DutyFree::Extensions._save_pending(to_be_saved)
                       # Try to find a unique item if one is referenced
                       # %%% There is possibility that when bringing in related classes using a nil
                       # in IMPORT_TEMPLATE[:all] that this will break.  Need to test deeply nested things.
                       start = (v.pre_prefix.blank? ? 0 : v.pre_prefix.length)
                       trim_prefix = v.titleize[start..-(v.name.length + 2)]
                       trim_prefix << ' ' unless trim_prefix.blank?
-                      klass = sub_next.klass
-                      # binding.pry if klass.name == 'OrderDetail'
-
                       # assoc.inverse_of is the belongs_to side of the has_many train we came in here on.
-                      sub_hm, criteria = klass.find_existing(uniques, cols, starred, import_template, keepers, assoc.inverse_of, row, sub_next, all, trim_prefix)
-
+                      sub_hm, criteria = assoc.klass.find_existing(uniques, cols, starred, import_template, keepers, assoc.inverse_of, row, sub_next, all, trim_prefix)
                       # If still not found then create a new related object using this has_many collection
-                      sub_next = sub_hm || sub_next.new(criteria)
+                      # (criteria.empty? ? nil : sub_next.new(criteria))
+                      if sub_hm
+                        sub_next = sub_hm
+                      elsif assoc.macro == :has_one
+                        bt_name = "#{assoc.inverse.name}="
+                        sub_next = assoc.klass.new(criteria)
+                        to_be_saved << [sub_next, sub_obj, bt_name, sub_next]
+                      else
+                        # Two other methods that are possible to check for here are :conditions and
+                        # :sanitized_conditions, which do not exist in Rails 4.0 and later.
+                        sub_next = if assoc.respond_to?(:require_association)
+                                     # With Rails < 4.0, sub_next could come in as an Array or a broken CollectionProxy
+                                     assoc.klass.new({ fk_from(assoc) => sub_obj.send(sub_obj.class.primary_key) }.merge(criteria))
+                                   else
+                                     sub_next.proxy_association.reflection.instance_variable_set(:@foreign_key, fk_from(assoc))
+                                     sub_next.new(criteria)
+                                   end
+                        to_be_saved << [sub_next]
+                      end
+                    end
+                    # Look for possible missing polymorphic detail
+                    # Maybe can test for this via assoc.through_reflection
+                    if assoc.is_a?(ActiveRecord::Reflection::ThroughReflection) &&
+                       (delegate = assoc.send(:delegate_reflection)&.active_record&.reflect_on_association(assoc.source_reflection_name)) &&
+                       delegate.options[:polymorphic]
+                      polymorphics << { parent: sub_next, child: sub_obj, type_col: delegate.foreign_type, id_col: delegate.foreign_key.to_s }
                     end
                     unless sub_next.nil?
                       # if sub_next.class.name == devise_class && # only for Devise users
@@ -343,14 +337,13 @@ module DutyFree
                       sub_objects[this_path] = sub_next if this_path.present?
                     end
                   end
-                  sub_obj = sub_next unless sub_next.nil?
+                  sub_obj = sub_next
                 end
                 next if sub_obj.nil?
 
-                sym = "#{v.name}=".to_sym
-                next unless sub_obj.respond_to?(sym)
+                next unless sub_obj.respond_to?(sym = "#{v.name}=".to_sym)
 
-                col_type = (sub_class = sub_obj.class).columns_hash[v.name.to_s]&.type
+                col_type = sub_obj.class.columns_hash[v.name.to_s]&.type
                 if col_type.nil? && (virtual_columns = import_template[:virtual_columns]) &&
                    (virtual_columns = virtual_columns[this_path] || virtual_columns)
                   col_type = virtual_columns[v.name]
@@ -358,9 +351,9 @@ module DutyFree
                 if col_type == :boolean
                   if row[key].nil?
                     # Do nothing when it's nil
-                  elsif %w[yes y].include?(row[key]&.downcase) # Used to cover 'true', 't', 'on'
+                  elsif %w[true t yes y].include?(row[key]&.strip&.downcase) # Used to cover 'on'
                     row[key] = true
-                  elsif %w[no n].include?(row[key]&.downcase) # Used to cover 'false', 'f', 'off'
+                  elsif %w[false f no n].include?(row[key]&.strip&.downcase) # Used to cover 'off'
                     row[key] = false
                   else
                     row_errors[v.name] ||= []
@@ -369,17 +362,18 @@ module DutyFree
                 end
                 sub_obj.send(sym, row[key])
                 # else
-                #   puts "  #{sub_class.name} doesn't respond to #{sym}"
+                #   puts "  #{sub_obj.class.name} doesn't respond to #{sym}"
               end
+              ::DutyFree::Extensions._save_pending(to_be_saved)
               # Try to save a final sub-object if one exists
-              sub_obj.save if sub_obj && this_path && !is_has_one && sub_obj.valid?
+              # sub_obj.save if sub_obj && this_path && !is_has_one && sub_obj.valid?
 
-              # Wire up has_one associations
-              has_ones.each do |hasone|
-                parent = sub_objects[hasone[0..-2].map(&:to_s).join(',')] || obj
-                hasone_object = sub_objects[hasone.map(&:to_s).join(',')]
-                parent.send("#{hasone[-1]}=", hasone_object) if parent.new_record? || hasone_object.valid?
-              end
+              # # Wire up has_one associations
+              # has_ones.each do |hasone|
+              #   parent = sub_objects[hasone[0..-2].map(&:to_s).join(',')] || obj
+              #   hasone_object = sub_objects[hasone.map(&:to_s).join(',')]
+              #   parent.send("#{hasone[-1]}=", hasone_object) if parent.new_record? || hasone_object.valid?
+              # end
 
               # Reinstate any missing polymorphic _type and _id values
               polymorphics.each do |poly|
@@ -416,26 +410,94 @@ module DutyFree
             s += v.last[1..-1].map { |line_num| { line_num => v.first } } if v.last.count > 1
             s
           end
-          # Check to see if they want to do anything after the import
           ret = { inserted: inserts, updated: updates, duplicates: duplicates, errors: errors }
-          if @after_import ||= (import_template[:after_import]) # || some generic after_import)
-            ret = ret2 if (ret2 = @after_import.call(ret)).is_a?(Hash)
+
+          # Check to see if they want to do anything after the import
+          # First if defined in the import_template, then if there is a method in the class,
+          # and finally (not yet implemented) a generic global after_import
+          my_after_import = import_template[:after_import]
+          my_after_import ||= respond_to?(:after_import) && method(:after_import)
+          # my_after_import ||= some generic my_after_import
+          if my_after_import
+            last_arg_idx = my_after_import.parameters.length - 1
+            arguments = [ret][0..last_arg_idx]
+            ret = ret2 if (ret2 = my_after_import.call(*arguments)).is_a?(Hash)
           end
         end
         ret
       end
 
+      def fk_from(assoc)
+        # Try first to trust whatever they've marked as being the foreign_key, and then look
+        # at the inverse's foreign key setting if available.  In all cases don't accept
+        # anything that's not backed with a real column in the table.
+        col_names = assoc.klass.column_names
+        if (fk_name = assoc.options[:foreign_key]) && col_names.include?(fk_name.to_s)
+          return fk_name
+        end
+        if (fk_name = assoc.inverse_of&.options&.fetch(:foreign_key) { nil }) &&
+           col_names.include?(fk_name.to_s)
+          return fk_name
+        end
+        if assoc.respond_to?(:foreign_key) && (fk_name = assoc.foreign_key) &&
+           col_names.include?(fk_name.to_s)
+          return fk_name
+        end
+        if (fk_name = assoc.inverse_of&.foreign_key) &&
+           col_names.include?(fk_name.to_s)
+          return fk_name
+        end
+        if (fk_name = assoc.inverse_of&.association_foreign_key) &&
+           col_names.include?(fk_name.to_s)
+          return fk_name
+        end
+        # Don't let this fool you -- we really are in search of the foreign key name here,
+        # and Rails 3.0 and older used some fairly interesting conventions, calling it instead
+        # the "primary_key_name"!
+        if assoc.respond_to?(:primary_key_name) && (fk_name = assoc.primary_key_name) &&
+           col_names.include?(fk_name.to_s)
+          return fk_name
+        end
+
+        puts "* Wow, had no luck at all finding a foreign key for #{assoc.inspect}"
+      end
+
       # For use with importing, based on the provided column list calculate all valid combinations
       # of unique columns.  If there is no valid combination, throws an error.
       # Returns an object found by this means.
-      def find_existing(uniques, cols, starred, import_template, keepers, train_we_came_in_here_on, row = nil, obj = nil, all = nil, trim_prefix = '')
-        col_name_offset = trim_prefix.length
-        @valid_uniques ||= {} # Fancy memoisation
+      def find_existing(uniques, cols, starred, import_template, keepers, train_we_came_in_here_on,
+                        row = nil, klass_or_collection = nil, template_all = nil, trim_prefix = '')
+        unless trim_prefix.blank?
+          cols = cols.map { |c| c.start_with?(trim_prefix) ? c[trim_prefix.length..-1] : nil }
+          starred = starred.each_with_object([]) do |v, s|
+            s << v[trim_prefix.length..-1] if v.start_with?(trim_prefix)
+            s
+          end
+        end
         col_list = cols.join('|')
-        unless (vus = @valid_uniques[col_list])
-          # Find all unique combinations that are available based on incoming columns, and
-          # pair them up with column number mappings.
-          template_column_objects = ::DutyFree::Extensions._recurse_def(self, all || import_template[:all], import_template).first
+
+        # First add in foreign key stuff we can find from belongs_to associations (other than the
+        # one we might have arrived here upon).
+        criteria = {} # Enough detail to find or build a new object
+        bt_criteria = {}
+        bt_criteria_all_nil = true
+        bt_col_indexes = []
+        available_bts = []
+        only_valid_uniques = (train_we_came_in_here_on == false)
+        uniq_lookups = {} # The data, or how to look up the data
+
+        vus = ((@valid_uniques ||= {})[col_list] ||= {}) # Fancy memoisation
+
+        # First, get an overall list of AVAILABLE COLUMNS before considering tricky foreign key stuff.
+        # ============================================================================================
+        # Generate a list of column names matched up with their zero-ordinal column number mapping for
+        # all columns from the incoming import data.
+        if (is_new_vus = vus.empty?)
+          template_column_objects = ::DutyFree::Extensions._recurse_def(
+            self,
+            template_all || import_template[:all],
+            import_template
+          ).first
           available = if trim_prefix.blank?
                         template_column_objects.select { |col| col.pre_prefix.blank? && col.prefix.blank? }
                       else
@@ -445,113 +507,190 @@ module DutyFree
                           trim_prefix_snake == "#{this_prefix}_"
                         end
                       end.map { |avail| avail.name.to_s.titleize }
-          vus = defined_uniques(uniques, cols, nil, starred).select do |k, _v|
-            is_good = true
-            k.each do |k_col|
-              unless k_col.start_with?(trim_prefix) && available.include?(k_col[col_name_offset..-1])
-                is_good = false
-                break
+        end
+
+        # Process FOREIGN KEY stuff by going through each belongs_to in this model.
+        # =========================================================================
+        # This list of all valid uniques will help to filter which foreign keys are kept, and also
+        # get further filtered later to arrive upon a final set of valid uniques.  (Often but not
+        # necessarily a specific valid unique as perhaps with a list of users you want to update some
+        # folks based on having their email as a unique identifier, and other folks by having a
+        # combination of their name and street address as unique, and use both of those possible
+        # unique variations to update phone numbers, and do that all as a part of one import.)
+        all_vus = defined_uniques(uniques, cols, col_list, starred, trim_prefix)
+
+        # %%% Ultimately may consider making this recursive
+        reflect_on_all_associations.each do |sn_bt|
+          next unless sn_bt.belongs_to? && (!train_we_came_in_here_on || sn_bt != train_we_came_in_here_on)
+
+          # # %%% Make sure there's a starred column we know about from this one
+          # uniq_lookups[sn_bt.foreign_key] = nil if only_valid_uniques
+
+          # This search prefix becomes something like "Order Details Product "
+          cols.each_with_index do |bt_col, idx|
+            next if bt_col_indexes.include?(idx) ||
+                    !bt_col&.start_with?(bt_prefix = (trim_prefix + "#{sn_bt.name.to_s.underscore.tr('_', ' ').titleize} "))
+
+            available_bts << bt_col
+            fk_id = if row
+                      # Max ID so if there are multiple matches, only the most recent one is picked.
+                      # %%% Need to stack these up in case there are multiple
+                      # (like first_name, last_name on a referenced employee)
+                      sn_bt.klass.where(keepers[idx].name => row[idx]).limit(1).pluck(MAX_ID).first
+                    else
+                      # elsif is_new_vus
+                      #   # Add to our criteria if this belongs_to is required
+                      #   bt_req_by_default = sn_bt.klass.respond_to?(:belongs_to_required_by_default) &&
+                      #                       sn_bt.klass.belongs_to_required_by_default
+                      #   unless !vus.values.first&.include?(idx) &&
+                      #          (sn_bt.options[:optional] || (sn_bt.options[:required] == false) || !bt_req_by_default)
+                      #     # # Add this fk to the criteria
+                      #     # criteria[fk_name] = fk_id
+
+                      #     ref = [keepers[idx].name, idx]
+                      #     # bt_criteria[(fk_name = sn_bt.foreign_key)] ||= [sn_bt.klass, []]
+                      #     # bt_criteria[fk_name].last << ref
+                      #     # bt_criteria[bt_col] = [sn_bt.klass, ref]
+
+                      #     # Maybe this is the most useful
+                      #     # First array is friendly column names, second is references
+                      #     foreign_uniques = (bt_criteria[sn_bt.name] ||= [sn_bt.klass, [], []])
+                      #     foreign_uniques[1] << ref
+                      #     foreign_uniques[2] << bt_col
+                      #     vus[bt_col] = foreign_uniques # And we can look up this growing set from any foreign column
+                      [sn_bt.klass, keepers[idx].name, idx]
+                    end
+            if fk_id
+              bt_col_indexes << idx
+              bt_criteria_all_nil = false
+            end
+            # If we're processing a row then this list of foreign key column name entries, named such as
+            # "order_id" or "product_id" instead of column-specific stuff like "Order Date" and "Product Name",
+            # is kept until the last and then gets merged on top of the other criteria before being returned.
+            bt_criteria[(fk_name = sn_bt.foreign_key)] = fk_id
+
+            # Check to see if belongs_tos are generally required on this specific table
+            bt_req_by_default = sn_bt.klass.respond_to?(:belongs_to_required_by_default) &&
+                                sn_bt.klass.belongs_to_required_by_default
+
+            # Add to our CRITERIA just the belongs_to things that check out.
+            # ==============================================================
+            # The first check, "all_vus.keys.first.none? { |k| k.start_with?(bt_prefix) }"
+            # is to see if one of the columns we're working with from the unique that we've chosen
+            # comes from the table referenced by this belongs_to (sn_bt).
+            #
+            # The second check on the :optional option and bt_req_by_default comes directly from
+            # how Rails 5 and later checks to see if a specific belongs_to is marked optional
+            # (or required), and without having that indication will fall back on checking the model
+            # itself to see if it requires belongs_tos by default.
+            next if all_vus.keys.first.none? { |k| k.start_with?(bt_prefix) } &&
+                    (sn_bt.options[:optional] || !bt_req_by_default)
+
+            # Add to the criteria
+            criteria[fk_name] = fk_id
+          end
+        end
+
+        # Now circle back find a final list of VALID UNIQUES by re-assessing the list of all valid uniques
+        # in relation to the available belongs_tos found in the last foreign key step.
+        if is_new_vus
+          available += available_bts
+          all_vus.each do |k, v|
+            combined_k = []
+            combined_v = []
+            k.each_with_index do |key, idx|
+              if available.include?(key)
+                combined_k << key
+                combined_v << v[idx]
               end
             end
-            is_good
+            vus[combined_k] = combined_v unless combined_k.empty?
           end
-          @valid_uniques[col_list] = vus
         end
 
+        # uniq_lookups = vus.inject({}) do |s, v|
+        #   return s if available_bts.include?(v.first) # These will be provided in criteria, and not uniq_lookups
+
+        #   # uniq_lookups[k[trim_prefix.length..-1].downcase.tr(' ', '_').to_sym] = val[idx] if k.start_with?(trim_prefix)
+        #   s[v.first.downcase.tr(' ', '_').to_sym] = v.last
+        #   s
+        # end
+
+        new_criteria_all_nil = bt_criteria_all_nil
+
         # Make sure they have at least one unique combination to take cues from
-        ret = {}
         unless vus.empty? # raise NoUniqueColumnError.new(I18n.t('import.no_unique_column_error'))
           # Convert the first entry to a simplified hash, such as:
           #   {[:investigator_institutions_name, :investigator_institutions_email] => [8, 9], ...}
           #     to {:name => 8, :email => 9}
-          key, val = vus.first
+          key, val = vus.first # Utilise the first identified set of valid uniques
           key.each_with_index do |k, idx|
-            ret[k[col_name_offset..-1].downcase.tr(' ', '_').to_sym] = val[idx] if k.start_with?(trim_prefix)
-          end
-        end
+            next if available_bts.include?(k) # These will be provided in criteria, and not uniq_lookups
 
-        # %%% If uniqueness is based on something else hanging out on a belongs_to then we're pretty hosed.
-        # (Case in point, importing Order with related Order Detail and Product, and then Product needs to
-        # be found or built first before OrderDetail.)
-        # Might have to do a deferred save kind of thing, and also make sure the product stuff came first
-        # before the other stuff
-
-        # Add in any foreign key stuff we can find from other belongs_to associations
-        # %%% This is starting to look like the other BelongsToAssociation code above around line
-        # 697, so it really needs to be turned into something recursive instead of this two-layer
-        # thick thing at best.
-        bts = reflect_on_all_associations.each_with_object([]) do |sn_assoc, s|
-          if sn_assoc.is_a?(ActiveRecord::Reflection::BelongsToReflection) &&
-             (!train_we_came_in_here_on || sn_assoc != train_we_came_in_here_on) &&
-             sn_assoc.klass != self # Omit stuff pointing to us (like self-referencing stuff)
-            # %%% Make sure there's a starred column we know about from this one
-            ret[sn_assoc.foreign_key] = nil if train_we_came_in_here_on == false
-            s << sn_assoc
-          end
-          s
-        end
-
-        # Find by all corresponding columns
-        criteria = {}
-        if train_we_came_in_here_on != false
-          criteria = ret.each_with_object({}) do |v, s|
-            s[v.first.to_sym] = row[v.last]
-            s
-          end
-        end
+            # uniq_lookups[k[trim_prefix.length..-1].downcase.tr(' ', '_').to_sym] = val[idx] if k.start_with?(trim_prefix)
+            k_sym = k.downcase.tr(' ', '_').to_sym
+            v = val[idx]
+            uniq_lookups[k_sym] = v # The column number in which to find the data
 
-        bts.each do |sn_bt|
-          # This search prefix becomes something like "Order Details Product "
-          cols.each_with_index do |bt_col, idx|
-            next unless bt_col.start_with?(trim_prefix + "#{sn_bt.name.to_s.underscore.tr('_', ' ').titleize} ")
+            next if only_valid_uniques || bt_col_indexes.include?(v)
 
-            fk_id = if row
-                      # Max ID so if there are multiple, only the most recent one is picked.
-                      # %%% Need to stack these up in case there are multiple (like first_name, last_name on a referenced employee)
-                      sn_bt.klass.where(keepers[idx].name => row[idx]).limit(1).pluck('MAX(id)').first
-                    else
-                      [sn_bt.klass, keepers[idx].name, idx]
-                    end
-            criteria[sn_bt.foreign_key] = fk_id
+            # Find by all corresponding columns
+            if (row_value = row[v])
+              new_criteria_all_nil = false
+              criteria[k_sym] = row_value # The data, or how to look up the data
+            end
           end
         end
 
-        # Short-circuiting this to only get back the valid_uniques?
-        return ret.merge(criteria) if train_we_came_in_here_on == false
-
-        # With this criteria, find any matching has_many row we can so we can update it
-        sub_hm = obj.find do |hm_obj|
-          is_good = true
-          criteria.each do |k, v|
-            if hm_obj.send(k).to_s != v.to_s
-              is_good = false
-              break
+        return uniq_lookups.merge(criteria) if only_valid_uniques
+
+        # If there's nothing to match upon then we're out
+        return [nil, {}] if new_criteria_all_nil
+
+        # With this criteria, find any matching has_many row we can so we can update it.
+        # First try directly looking it up through ActiveRecord.
+        found_object = klass_or_collection.find_by(criteria)
+        # If not successful, such as when fields are exposed via helper methods instead of being
+        # real columns in the database tables, try this more intensive approach.  This is useful
+        # if you had full name kind of data coming in on a spreadsheeet, but in the destination
+        # table it's broken out to first_name, middle_name, surname.  By writing both full_name
+        # and full_name= methods, the importer can check to see if this entry is already there,
+        # and put a new row in if not, having one incoming name break out to populate three
+        # destination columns.
+        unless found_object || klass_or_collection.is_a?(Array)
+          found_object = klass_or_collection.find do |obj|
+            is_good = true
+            criteria.each do |k, v|
+              if obj.send(k).to_s != v.to_s
+                is_good = false
+                break
+              end
             end
+            is_good
           end
-          is_good
         end
-        # Try looking it up through ActiveRecord
-        # %%% Should we perhaps do this first before the more intensive find routine above?
-        sub_hm = obj.find_by(criteria) if sub_hm.nil?
-        [sub_hm, criteria]
+        # Standard criteria as well as foreign key column name detail with exact foreign keys
+        # that match up to a primary key so that if needed a new related object can be built,
+        # complete with all its association detail.
+        [found_object, criteria.merge(bt_criteria)]
       end
 
     private
 
-      def defined_uniques(uniques, cols = [], col_list = nil, starred = [])
+      def defined_uniques(uniques, cols = [], col_list = nil, starred = [], trim_prefix = '')
         col_list ||= cols.join('|')
-        @defined_uniques ||= {}
-        unless (defined_uniq = @defined_uniques[col_list])
+        unless (defined_uniq = (@defined_uniques ||= {})[col_list])
           utilised = {} # Track columns that have been referenced thusfar
           defined_uniq = uniques.each_with_object({}) do |unique, s|
             if unique.is_a?(Array)
               key = []
               value = []
               unique.each do |unique_part|
-                val = cols.index(unique_part_name = unique_part.to_s.titleize)
-                next if val.nil?
+                val = (unique_part_name = unique_part.to_s.titleize).start_with?(trim_prefix) &&
+                      cols.index(upn = unique_part_name[trim_prefix.length..-1])
+                next unless val
 
-                key << unique_part_name
+                key << upn
                 value << val
               end
               unless key.empty?
@@ -559,29 +698,60 @@ module DutyFree
                 utilised[key] = nil
               end
             else
-              val = cols.index(unique_part_name = unique.to_s.titleize)
-              unless val.nil?
-                s[[unique_part_name]] = [val]
-                utilised[[unique_part_name]] = nil
+              val = (unique_name = unique.to_s.titleize).start_with?(trim_prefix) &&
+                    cols.index(un = unique_name[trim_prefix.length..-1])
+              if val
+                s[[un]] = [val]
+                utilised[[un]] = nil
               end
             end
+            s
+          end
+          if defined_uniq.empty?
+            (starred - utilised.keys).each { |star| defined_uniq[[star]] = [cols.index(star)] }
+            # %%% puts "Tried to establish #{defined_uniq.inspect}"
           end
-          (starred - utilised.keys).each { |star| defined_uniq[[star]] = [cols.index(star)] }
           @defined_uniques[col_list] = defined_uniq
         end
         defined_uniq
       end
     end # module ClassMethods
 
+    # Called before building any object linked through a has_one or has_many so that foreign key IDs
+    # can be added properly to those new objects.  Finally at the end also called to save everything.
+    def self._save_pending(to_be_saved)
+      while (tbs = to_be_saved.pop)
+        ais = (tbs.first.class.respond_to?(:around_import_save) && tbs.first.class.method(:around_import_save)) ||
+              (respond_to?(:around_import_save) && method(:around_import_save))
+        if ais
+          # Send them the sub_obj even if it might be invalid so they can choose
+          # to make it valid if they wish.
+          ais.call(tbs.first) do |modded_obj = nil|
+            modded_obj = (modded_obj || tbs.first)
+            modded_obj.save if modded_obj&.valid?
+          end
+        elsif tbs.first.valid?
+          tbs.first.save
+        else
+          puts "* Unable to save #{tbs.first.inspect}"
+        end
+        # puts "Save #{tbs.first.class.name} #{tbs.first&.id} #{!tbs.first.new_record?}"
+        unless tbs[1].nil? || tbs.first.new_record?
+          # puts "Calling #{tbs[1].class.name} #{tbs[1]&.id} .#{tbs[2]} #{tbs[3].class.name} #{tbs[3]&.id}"
+          tbs[1].send(tbs[2], tbs[3])
+        end
+      end
+    end
+
     # The snake-cased column alias names used in the query to export data
     def self._template_columns(klass, import_template = nil)
       template_detail_columns = klass.instance_variable_get(:@template_detail_columns)
-      if (template_import_columns = klass.instance_variable_get(:@template_import_columns)) != import_template
-        klass.instance_variable_set(:@template_import_columns, template_import_columns = import_template)
+      if klass.instance_variable_get(:@template_import_columns) != import_template
+        klass.instance_variable_set(:@template_import_columns, import_template)
         klass.instance_variable_set(:@template_detail_columns, (template_detail_columns = nil))
       end
       unless template_detail_columns
-        puts "* Redoing *"
+        # puts "* Redoing *"
         template_detail_columns = _recurse_def(klass, import_template[:all], import_template).first.map(&:to_sym)
         klass.instance_variable_set(:@template_detail_columns, template_detail_columns)
       end
@@ -590,16 +760,21 @@ module DutyFree
 
     # Recurse and return two arrays -- one with all columns in sequence, and one a hierarchy of
     # nested hashes to be used with ActiveRecord's .joins() to facilitate export.
-    def self._recurse_def(klass, array, import_template, assocs = [], joins = [], pre_prefix = '', prefix = '')
+    def self._recurse_def(klass, array, import_template, order_by = [], assocs = [], joins = [], pre_prefix = '', prefix = '')
+      prefixes = ::DutyFree::Util._prefix_join([pre_prefix, prefix])
       # Confirm we can actually navigate through this association
       prefix_assoc = (assocs.last&.klass || klass).reflect_on_association(prefix) if prefix.present?
-      assocs = assocs.dup << prefix_assoc unless prefix_assoc.nil?
-      prefixes = ::DutyFree::Util._prefix_join([pre_prefix, prefix])
+      if prefix_assoc
+        assocs = assocs.dup << prefix_assoc
+        if prefix_assoc.macro == :has_many && (pk = prefix_assoc.active_record.primary_key)
+          order_by << ["#{prefixes.tr('.', '_')}_", pk]
+        end
+      end
       array = array.inject([]) do |s, col|
         s + if col.is_a?(Hash)
               col.inject([]) do |s2, v|
                 joins << { v.first.to_sym => (joins_array = []) }
-                s2 += _recurse_def(klass, (v.last.is_a?(Array) ? v.last : [v.last]), import_template, assocs, joins_array, prefixes, v.first.to_sym).first
+                s2 + _recurse_def(klass, (v.last.is_a?(Array) ? v.last : [v.last]), import_template, order_by, assocs, joins_array, prefixes, v.first.to_sym).first
               end
             elsif col.nil?
               if assocs.empty?
@@ -608,7 +783,7 @@ module DutyFree
                 # Bring in from another class
                 joins << { prefix => (joins_array = []) }
                 # %%% Also bring in uniques and requireds
-                _recurse_def(klass, assocs.last.klass::IMPORT_TEMPLATE[:all], import_template, assocs, joins_array, prefixes).first
+                _recurse_def(klass, assocs.last.klass::IMPORT_TEMPLATE[:all], import_template, order_by, assocs, joins_array, prefixes).first
               end
             else
               [::DutyFree::Column.new(col, pre_prefix, prefix, assocs, klass, import_template[:as])]
@@ -618,9 +793,13 @@ module DutyFree
     end
   end # module Extensions
 
-  class NoUniqueColumnError < ActiveRecord::RecordNotUnique
+  # Rails < 4.0 doesn't have ActiveRecord::RecordNotUnique, so use the more generic ActiveRecord::ActiveRecordError instead
+  ar_not_unique_error = ActiveRecord.const_defined?('RecordNotUnique') ? ActiveRecord::RecordNotUnique : ActiveRecord::ActiveRecordError
+  class NoUniqueColumnError < ar_not_unique_error
   end
 
-  class LessThanHalfAreMatchingColumnsError < ActiveRecord::RecordInvalid
+  # Rails < 4.2 doesn't have ActiveRecord::RecordInvalid, so use the more generic ActiveRecord::ActiveRecordError instead
+  ar_invalid_error = ActiveRecord.const_defined?('RecordInvalid') ? ActiveRecord::RecordInvalid : ActiveRecord::ActiveRecordError
+  class LessThanHalfAreMatchingColumnsError < ar_invalid_error
   end
 end