duty_free 1.0.8 → 1.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 714b772aa09eb15409cd7c58d4ccb64f967caf41f2a13e7813e90cc94d7eb48f
-  data.tar.gz: 750dfe36b729979a0356f14f8588568822ae22911c142d294358231504daa78e
+  metadata.gz: dc07e472cca2426a694a59754a83d12b93ad6572530d9b5f584b8aecfbbbacf4
+  data.tar.gz: 7b160b28ab7e70409ee095d6ba1892fa06d24310ebba4529ae0a5f8c57949673
 SHA512:
-  metadata.gz: 9d77a228b4e4990dad39882a4e67d1009f3c16114bc0e9b24280e1f8e9437c7fc55a8884eb37e8f6c15898958833c7f79bb7984d721f4dbd1fce7311ca6421d0
-  data.tar.gz: 1d35f3197ab500a6210eecc561190c02a04d87e3d4b2634a9a62695488d4e2ed0825f7240836aa9d32fe8ce01e06be38ddccda1955ca3d3669fd7ad18e8335f2
+  metadata.gz: 25e0702b19e9e5e810f0632da354060ae15da6207c7af33c930eb5825c2f28046e09a481a49ce95bbce46ddb02cc6dd991d2d2728582335f1260b7153bf28998
+  data.tar.gz: 0e265c33a7b1611aa8cb5c28edef6632d3f9e1c2adc9c15f644b6757b2229fb22248a0a37589449e97adbea8876e76aa601331b44556f277ea17255ad6af47d0

data/lib/duty_free/extensions.rb

@@ -9,6 +9,7 @@ module DutyFree
   # rubocop:disable Style/CommentedKeyword
   module Extensions
     MAX_ID = Arel.sql('MAX(id)')
+    IS_AMOEBA = Gem.loaded_specs['amoeba']
 
     def self.included(base)
       base.send :extend, ClassMethods
@@ -111,8 +112,8 @@ module DutyFree
         rows
       end
 
-      def df_import(data, import_template = nil)
-        ::DutyFree::Extensions.import(self, data, import_template)
+      def df_import(data, import_template = nil, insert_only = false)
+        ::DutyFree::Extensions.import(self, data, import_template, insert_only)
       end
 
     private
@@ -159,7 +160,7 @@ module DutyFree
       # For use with importing, based on the provided column list calculate all valid combinations
       # of unique columns.  If there is no valid combination, throws an error.
       # Returns an object found by this means, as well as the criteria that was used to find it.
-      def _find_existing(uniques, cols, starred, import_template, keepers, train_we_came_in_here_on,
+      def _find_existing(uniques, cols, starred, import_template, keepers, train_we_came_in_here_on, insert_only,
                          row = nil, klass_or_collection = nil, template_all = nil, trim_prefix = '',
                          assoc = nil, base_obj = nil)
         unless trim_prefix.blank?
@@ -262,7 +263,8 @@ module DutyFree
             # If we're processing a row then this list of foreign key column name entries, named such as
             # "order_id" or "product_id" instead of column-specific stuff like "Order Date" and "Product Name",
             # is kept until the last and then gets merged on top of the other criteria before being returned.
-            bt_criteria[(fk_name = sn_bt.foreign_key)] = fk_id
+            fk_name = sn_bt.foreign_key
+            bt_criteria[fk_name] = fk_id unless bt_criteria.include?(fk_name)
 
             # Check to see if belongs_tos are generally required on this specific table
             bt_req_by_default = sn_bt.klass.respond_to?(:belongs_to_required_by_default) &&
@@ -282,7 +284,7 @@ module DutyFree
                     (sn_bt.options[:optional] || !bt_req_by_default)
 
             # Add to the criteria
-            criteria[fk_name] = fk_id
+            criteria[fk_name] = fk_id if insert_only && !criteria.include?(fk_name)
           end
         end
 
@@ -332,14 +334,15 @@ module DutyFree
             # Find by all corresponding columns
             if (row_value = row[v])
               new_criteria_all_nil = false
-              criteria[k_sym] = row_value # The data, or how to look up the data
+              criteria[k_sym] = row_value # The data
             end
           end
         end
+        # puts uniq_lookups.inspect
 
-        return uniq_lookups.merge(criteria) if only_valid_uniques
+        return [uniq_lookups.merge(criteria), bt_criteria] if only_valid_uniques
         # If there's nothing to match upon then we're out
-        return [nil, {}] if new_criteria_all_nil
+        return [nil, {}, {}] if new_criteria_all_nil
 
         # With this criteria, find any matching has_many row we can so we can update it.
         # First try directly looking it up through ActiveRecord.
@@ -410,12 +413,12 @@ module DutyFree
         # Standard criteria as well as foreign key column name detail with exact foreign keys
         # that match up to a primary key so that if needed a new related object can be built,
         # complete with all its association detail.
-        [found_object, criteria.merge(bt_criteria)]
+        [found_object, criteria, bt_criteria]
       end # _find_existing
     end # module ClassMethods
 
     # With an array of incoming data, the first row having column names, perform the import
-    def self.import(obj_klass, data, import_template = nil)
+    def self.import(obj_klass, data, import_template = nil, insert_only)
       instance_variable_set(:@defined_uniques, nil)
       instance_variable_set(:@valid_uniques, nil)
 
@@ -559,7 +562,7 @@ module DutyFree
             raise ::DutyFree::LessThanHalfAreMatchingColumnsError, I18n.t('import.altered_import_template_coumns') if keepers.length < (cols.length / 2) - 1
 
             # Returns just the first valid unique lookup set if there are multiple
-            valid_unique = obj_klass.send(:_find_existing, uniques, cols, starred, import_template, keepers, false)
+            valid_unique, bt_criteria = obj_klass.send(:_find_existing, uniques, cols, starred, import_template, keepers, false, insert_only)
             # Make a lookup from unique values to specific IDs
             existing = obj_klass.pluck(*([:id] + valid_unique.keys)).each_with_object(existing) do |v, s|
               s[v[1..-1].map(&:to_s)] = v.first
@@ -578,16 +581,17 @@ module DutyFree
             to_be_saved = []
             # Check to see if they want to preprocess anything
             existing_unique = @before_process.call(valid_unique, existing_unique) if @before_process ||= import_template[:before_process]
-            if (criteria = existing[existing_unique])
-              obj = obj_klass.find(criteria)
-            else
-              is_insert = true
-              # unless build_tables.empty? # include?()
-              #   binding.pry
-              #   x = 5
-              # end
-              to_be_saved << [obj = obj_klass.new]
-            end
+            obj = if (criteria = existing[existing_unique])
+                    obj_klass.find(criteria)
+                  else
+                    is_insert = true
+                    # unless build_tables.empty? # include?()
+                    #   binding.pry
+                    #   x = 5
+                    # end
+                    obj_klass.new
+                  end
+            to_be_saved << [obj] unless criteria # || this one has any belongs_to that will be modified here
             sub_obj = nil
             polymorphics = []
             sub_objects = {}
@@ -623,18 +627,22 @@ module DutyFree
                   start = 0
                   trim_prefix = v.titleize[start..-(v.name.length + 2)]
                   trim_prefix << ' ' unless trim_prefix.blank?
-                  if (sub_next = sub_obj.send(path_part)).nil? && assoc.belongs_to?
+                  binding.pry unless assoc
+                  if assoc.belongs_to?
                     klass = Object.const_get(assoc&.class_name)
                     # Try to find a unique item if one is referenced
                     sub_next = nil
                     begin
-                      sub_next, criteria = klass.send(:_find_existing, uniques, cols, starred, import_template, keepers, nil, row, klass, all, trim_prefix, assoc)
+                      sub_next, criteria, bt_criteria = klass.send(:_find_existing, uniques, cols, starred, import_template, keepers, nil,
+                                                                   false, # insert_only
+                                                                   row, klass, all, trim_prefix, assoc)
                     rescue ::DutyFree::NoUniqueColumnError
                     end
-                    # puts "#{v.path} #{criteria.inspect}"
                     bt_name = "#{path_part}="
-                    unless sub_next || (klass == sub_obj.class && criteria.empty?)
-                      sub_next = klass.new(criteria || {})
+                    # Not yet wired up to the right one, or going to the parent of a self-referencing model?
+                    # puts "#{v.path} #{criteria.inspect}"
+                    unless sub_next || (klass == sub_obj.class && (all_criteria = criteria.merge(bt_criteria)).empty?)
+                      sub_next = klass.new(all_criteria || {})
                       to_be_saved << [sub_next, sub_obj, bt_name, sub_next]
                     end
                     # This wires it up in memory, but doesn't yet put the proper foreign key ID in
@@ -654,15 +662,17 @@ module DutyFree
                   # Rails 4.0 and later can do:  sub_next.is_a?(ActiveRecord::Associations::CollectionProxy)
                   elsif [:has_many, :has_one, :has_and_belongs_to_many].include?(assoc.macro) # && !assoc.options[:through]
                     ::DutyFree::Extensions._save_pending(to_be_saved)
+                    sub_next = sub_obj.send(path_part)
                     # Try to find a unique item if one is referenced
                     # %%% There is possibility that when bringing in related classes using a nil
                     # in IMPORT_TEMPLATE[:all] that this will break.  Need to test deeply nested things.
 
                     # assoc.inverse_of is the belongs_to side of the has_many train we came in here on.
-                    sub_hm, criteria = assoc.klass.send(:_find_existing, uniques, cols, starred, import_template, keepers, assoc.inverse_of,
-                                                        row, sub_next, all, trim_prefix, assoc,
-                                                        # Just in case we're running Rails < 4.0 and this is a haas_*
-                                                        sub_obj)
+                    sub_hm, criteria, bt_criteria = assoc.klass.send(:_find_existing, uniques, cols, starred, import_template, keepers, assoc.inverse_of,
+                                                                     false, # insert_only
+                                                                     row, sub_next, all, trim_prefix, assoc,
+                                                                     # Just in case we're running Rails < 4.0 and this is a has_*
+                                                                     sub_obj)
                     # If still not found then create a new related object using this has_many collection
                     # (criteria.empty? ? nil : sub_next.new(criteria))
                     if sub_hm

data/lib/duty_free/suggest_template.rb

@@ -7,7 +7,6 @@ module DutyFree
       # Helpful suggestions to get started creating a template
       # Pass in -1 for hops if you want to traverse all possible links
       def suggest_template(hops = 0, do_has_many = false, show_output = true, this_klass = self)
-        ::DutyFree.instance_variable_set(:@errored_assocs, [])
         ::DutyFree.instance_variable_set(:@errored_columns, [])
         uniques, _required = ::DutyFree::SuggestTemplate._suggest_unique_column(this_klass, nil, '')
         template, required = ::DutyFree::SuggestTemplate._suggest_template(hops, do_has_many, this_klass)
@@ -17,7 +16,6 @@ module DutyFree
           all: template,
           as: {}
         }
-        # puts "Errors: #{::DutyFree.instance_variable_get(:@errored_assocs).inspect}"
 
         if show_output
           path = this_klass.name.split('::').map(&:underscore).join('/')
@@ -32,161 +30,369 @@ module DutyFree
       end
     end
 
-    def self._suggest_template(hops, do_has_many, this_klass, poison_links = [], path = '')
-      errored_assocs = ::DutyFree.instance_variable_get(:@errored_assocs)
-      this_primary_key = Array(this_klass.primary_key)
-      # Find all associations, and track all belongs_tos
-      this_belongs_tos = []
-      assocs = {}
-      this_klass.reflect_on_all_associations.each do |assoc|
-        # PolymorphicReflection AggregateReflection RuntimeReflection
-        is_belongs_to = assoc.belongs_to?
-        # Figure out if it's belongs_to, has_many, or has_one
-        belongs_to_or_has_many =
-          if is_belongs_to
-            'belongs_to'
-          elsif (is_habtm = assoc.macro == :has_and_belongs_to_many)
-            'has_and_belongs_to_many'
-          elsif assoc.macro == :has_many
-            'has_many'
-          else
-            'has_one'
+    def self._suggest_template(hops, do_has_many, this_klass)
+      poison_links = []
+      requireds = []
+      errored_assocs = []
+      buggy_bts = []
+      is_papertrail = Object.const_defined?('::PaperTrail::Version')
+      # Get a list of all polymorphic models by searching out every has_many that has an :as option
+      all_polymorphics = Hash.new { |h, k| h[k] = [] }
+      _all_models.each do |model|
+        model.reflect_on_all_associations.select do |assoc|
+          # If the model name can not be found then we get something like this here:
+          # 1: from /home/lorin/.rvm/gems/ruby-2.6.5/gems/activerecord-5.2.4.6/lib/active_record/reflection.rb:418:in `compute_class'
+          # /home/lorin/.rvm/gems/ruby-2.6.5/gems/activerecord-5.2.4.6/lib/active_record/inheritance.rb:196:in `compute_type': uninitialized constant Site::SiteTeams (NameError)
+          ret = nil
+          begin
+            ret = assoc.polymorphic? ||
+                  (assoc.options.include?(:as) && !(is_papertrail && assoc.klass&.<=(PaperTrail::Version)))
+          rescue NameError
+            false
           end
-        # Always process belongs_to, and also process has_one and has_many if do_has_many is chosen.
-        # Skip any HMT or HABTM. (Maybe break out HABTM into a combo HM and BT in the future.)
-        if is_habtm
-          puts "* In the #{this_klass.name} model there's a problem with:  \"has_and_belongs_to_many :#{assoc.name}\"  because join table \"#{assoc.join_table}\" does not exist.  You can create it with a create_join_table migration." unless ActiveRecord::Base.connection.table_exists?(assoc.join_table)
-          # %%% Search for other associative candidates to use instead of this HABTM contraption
-          puts "* In the #{this_klass.name} model there's a problem with:  \"has_and_belongs_to_many :#{assoc.name}\"  because it includes \"through: #{assoc.options[:through].inspect}\" which is pointless and should be removed." if assoc.options.include?(:through)
-        end
-        if (is_through = assoc.is_a?(ActiveRecord::Reflection::ThroughReflection)) && assoc.options.include?(:as)
-          puts "* In the #{this_klass.name} model there's a problem with:  \"has_many :#{assoc.name} through: #{assoc.options[:through].inspect}\"  because it also includes \"as: #{assoc.options[:as].inspect}\", so please choose either for this line to be a \"has_many :#{assoc.name} through:\" or to be a polymorphic \"has_many :#{assoc.name} as:\".  It can't be both."
+          ret
+        end.each do |assoc|
+          poly_hm = if assoc.belongs_to?
+                      poly_key = [model, assoc.name.to_sym] if assoc.polymorphic?
+                      nil
+                    else
+                      poly_key = [assoc.klass, assoc.options[:as]]
+                      [model, assoc.macro, assoc.name.to_sym]
+                    end
+          next if all_polymorphics.include?(poly_key) && all_polymorphics[poly_key].include?(poly_hm)
+
+          poly_hms = all_polymorphics[poly_key]
+          poly_hms << poly_hm if poly_hm
         end
-        next if is_through || is_habtm || (!is_belongs_to && !do_has_many) || errored_assocs.include?(assoc)
+      end
 
-        if is_belongs_to && assoc.options[:polymorphic] # Polymorphic belongs_to?
-          # Load all models
-          # %%% Note that this works in Rails 5.x, but may not work in Rails 6.0 and later, which uses the Zeitwerk loader by default:
-          Rails.configuration.eager_load_namespaces.select { |ns| ns < Rails::Application }.each(&:eager_load!)
-          # Find all current possible polymorphic relations
-          ActiveRecord::Base.descendants.each do |model|
-            # Skip auto-generated HABTM_DestinationModel models
-            next if model.respond_to?(:table_name_resolver) &&
-                    model.name.start_with?('HABTM_') &&
-                    model.table_name_resolver.is_a?(
-                      ActiveRecord::Associations::Builder::HasAndBelongsToMany::JoinTableResolver::KnownClass
-                    )
+      bad_polymorphic_hms = []
+      # "path" starts with ''
+      # "template" starts with [], and we hold on to the root piece here so we can return it after
+      # going through all the layers, at which point it will have grown to include the entire hierarchy.
+      this_layer = [[do_has_many, this_klass, '', (whole_template = [])]]
+      loop do
+        next_layer = []
+        this_layer.each do |klass_item|
+          do_has_many, this_klass, path, template = klass_item
+          this_primary_key = Array(this_klass.primary_key)
+          # Find all associations, and track all belongs_tos
+          this_belongs_tos = []
+          assocs = {}
+          this_klass.reflect_on_all_associations.each do |assoc|
+            # PolymorphicReflection AggregateReflection RuntimeReflection
+            is_belongs_to = assoc.belongs_to?
+            # Figure out if it's belongs_to, has_many, or has_one
+            belongs_to_or_has_many =
+              if is_belongs_to
+                'belongs_to'
+              elsif (is_habtm = assoc.macro == :has_and_belongs_to_many)
+                'has_and_belongs_to_many'
+              elsif assoc.macro == :has_many
+                'has_many'
+              else
+                'has_one'
+              end
+            # Always process belongs_to, and also process has_one and has_many if do_has_many is chosen.
+            # Skip any HMT or HABTM. (Maybe break out HABTM into a combo HM and BT in the future.)
+            if is_habtm
+              puts "* #{this_klass.name} model - problem with:  \"has_and_belongs_to_many :#{assoc.name}\"  because join table \"#{assoc.join_table}\" does not exist.  You can create it with a create_join_table migration." unless ActiveRecord::Base.connection.table_exists?(assoc.join_table)
+              # %%% Search for other associative candidates to use instead of this HABTM contraption
+              puts "* #{this_klass.name} model - problem with:  \"has_and_belongs_to_many :#{assoc.name}\"  because it includes \"through: #{assoc.options[:through].inspect}\" which is pointless and should be removed." if assoc.options.include?(:through)
+            end
+            if (is_through = assoc.is_a?(ActiveRecord::Reflection::ThroughReflection)) && assoc.options.include?(:as)
+              puts "* #{this_klass.name} model - problem with:  \"has_many :#{assoc.name} through: #{assoc.options[:through].inspect}\"  because it also includes \"as: #{assoc.options[:as].inspect}\", " \
+                   "so please choose either for this line to be a \"has_many :#{assoc.name} through:\" or to be a polymorphic \"has_many :#{assoc.name} as:\".  It can't be both."
+            end
+            next if is_through || is_habtm || (!is_belongs_to && !do_has_many) || errored_assocs.include?(assoc)
 
-            # Find applicable polymorphic has_many associations from each real model
-            model.reflect_on_all_associations.each do |poly_assoc|
-              next unless poly_assoc.macro == :has_many && poly_assoc.inverse_of == assoc
+            # Polymorphic belongs_to?
+            # (checking all_polymorphics in order to handle the rare case when the belongs_to side of a polymorphic association is missing  "polymorphic: true")
+            if is_belongs_to && (
+                                  assoc.options[:polymorphic] ||
+                                  (
+                                    all_polymorphics.include?(poly_key = [assoc.active_record, assoc.name.to_sym]) &&
+                                    all_polymorphics[poly_key].map { |p| p&.first }.include?(this_klass)
+                                  )
+                                )
+              # Find all current possible polymorphic relations
+              _all_models.each do |model|
+                # Skip auto-generated HABTM_DestinationModel models
+                next if model.respond_to?(:table_name_resolver) &&
+                        model.name.start_with?('HABTM_') &&
+                        model.table_name_resolver.is_a?(
+                          ActiveRecord::Associations::Builder::HasAndBelongsToMany::JoinTableResolver::KnownClass
+                        )
 
-              this_belongs_tos += (fkeys = [poly_assoc.type, poly_assoc.foreign_key])
-              assocs["#{assoc.name}_#{poly_assoc.active_record.name.underscore}".to_sym] = [[fkeys, assoc.active_record], poly_assoc.active_record]
-            end
-          end
-        else
-          # Is it a polymorphic has_many, which is defined using as: :somethingable ?
-          is_polymorphic_hm = assoc.inverse_of&.options&.fetch(:polymorphic) { nil }
-          begin
-            # Standard has_one, or has_many, and belongs_to uses assoc.klass.
-            # Also polymorphic belongs_to uses assoc.klass.
-            assoc_klass = is_polymorphic_hm ? assoc.inverse_of.active_record : assoc.klass
-          rescue NameError # For models which cannot be found by name
-          end
-          new_assoc =
-            if assoc_klass.nil?
-              puts "* In the #{this_klass.name} model there's a problem with:  \"#{belongs_to_or_has_many} :#{assoc.name}\"  because there is no \"#{assoc.class_name}\" model."
-              nil # Cause this one to be excluded
-            elsif is_belongs_to
-              this_belongs_tos << (fk = assoc.foreign_key.to_s)
-              [[[fk], assoc.active_record], assoc_klass]
-            else # has_many or has_one
-              inverse_foreign_keys = is_polymorphic_hm ? [assoc.type, assoc.foreign_key] : [assoc.inverse_of&.foreign_key&.to_s]
-              missing_key_columns = inverse_foreign_keys - assoc_klass.columns.map(&:name)
-              if missing_key_columns.empty?
-                puts "* Missing inverse foreign key for #{this_klass.name} #{belongs_to_or_has_many} :#{assoc.name}" if inverse_foreign_keys.first.nil?
-                # puts "Has columns #{inverse_foreign_keys.inspect}"
-                [[inverse_foreign_keys, assoc_klass], assoc_klass]
-              else
-                if inverse_foreign_keys.length > 1
-                  puts "* The #{assoc_klass.name} model is missing #{missing_key_columns.join(' and ')} columns to allow it to support polymorphic inheritance."
-                else
-                  print "* In the #{this_klass.name} model there's a problem with:  \"#{belongs_to_or_has_many} :#{assoc.name}\"."
+                # Find applicable polymorphic has_many associations from each real model
+                model.reflect_on_all_associations.each do |poly_assoc|
+                  next unless [:has_many, :has_one].include?(poly_assoc.macro) && poly_assoc.inverse_of == assoc
 
-                  if (inverses = _find_belongs_tos(assoc_klass, this_klass, errored_assocs)).empty?
-                    if inverse_foreign_keys.first.nil?
-                      puts "  Consider adding \"foreign_key: :#{this_klass.name.underscore}_id\" regarding some column in #{assoc_klass.name} to this #{belongs_to_or_has_many} entry."
-                    else
-                      puts "  (Cannot find foreign key \"#{inverse_foreign_keys.first.inspect}\" in #{assoc_klass.name}.)"
+                  this_belongs_tos += (fkeys = [poly_assoc.type, poly_assoc.foreign_key])
+                  assocs["#{assoc.name}_#{poly_assoc.active_record.name.underscore}".to_sym] = [[fkeys, assoc.active_record], poly_assoc.active_record]
+                end
+              end
+            else
+              # Is it a polymorphic has_many, which is defined using as: :somethingable ?
+              is_polymorphic_hm = assoc.inverse_of&.options&.fetch(:polymorphic) { nil }
+              begin
+                # Standard has_one, or has_many, and belongs_to uses assoc.klass.
+                # Also polymorphic belongs_to uses assoc.klass.
+                assoc_klass = is_polymorphic_hm ? assoc.inverse_of.active_record : assoc.klass
+              rescue NameError # For models which cannot be found by name
+              end
+              # Skip any PaperTrail audited things
+              # rubocop:disable Lint/SafeNavigationConsistency
+              next if (Object.const_defined?('PaperTrail::Version') && assoc_klass&.<=(PaperTrail::Version) && assoc.options.include?(:as)) ||
+                      # And any goofy self-referencing aliases
+                      (!is_belongs_to && assoc_klass <= assoc.active_record && assoc.foreign_key.to_s == assoc.active_record.primary_key)
+
+              # rubocop:enable Lint/SafeNavigationConsistency
+
+              # Avoid getting goofed up by the belongs_to side of a broken polymorphic association
+              assoc_klass = nil if assoc.belongs_to? && !(assoc_klass <= ActiveRecord::Base)
+
+              if !is_polymorphic_hm && assoc.options.include?(:as)
+                assoc_klass = assoc.inverse_of.active_record
+                is_polymorphic_hm = true
+                bad_polymorphic_hm = [assoc_klass, assoc.inverse_of]
+                unless bad_polymorphic_hms.include?(bad_polymorphic_hm)
+                  bad_polymorphic_hms << bad_polymorphic_hm
+                  puts "* #{assoc_klass.name} model - problem with the polymorphic association  \"belongs_to :#{assoc.inverse_of.name}\".  You can fix this in one of two ways:"
+                  puts '  (1) add "polymorphic: true" on this belongs_to line, or'
+                  poly_hms = all_polymorphics.inject([]) do |s, poly_hm|
+                    if (key = poly_hm.first).first <= assoc_klass && key.last == assoc.inverse_of.name
+                      s += poly_hm.last
                     end
+                    s
+                  end
+                  puts "  (2) Undo #{assoc_klass.name} polymorphism by removing  \"as: :#{assoc.inverse_of.name}\"  in these #{poly_hms.length} places:"
+                  poly_hms.each { |poly_hm| puts "      In the #{poly_hm.first.name} class from the line:  #{poly_hm[1]} :#{poly_hm.last}" }
+                end
+              end
+
+              new_assoc =
+                if assoc_klass.nil?
+                  # In case this is a buggy polymorphic belongs_to, keep track of all of these and at the very end
+                  # only show the pertinent ones.
+                  if is_belongs_to
+                    buggy_bts << [this_klass, assoc]
                   else
-                    puts "  Consider adding \"#{inverses.map { |x| "inverse_of: :#{x.name}" }.join(' or ')}\" to this entry."
+                    puts "* #{this_klass.name} model - problem with:  \"#{belongs_to_or_has_many} :#{assoc.name}\"  because there is no \"#{assoc.class_name}\" model."
                   end
+                  nil # Cause this one to be excluded
+                elsif is_belongs_to
+                  this_belongs_tos << (foreign_key = assoc.foreign_key.to_s)
+                  [[[foreign_key], assoc.active_record], assoc_klass]
+                elsif _all_tables.include?(assoc_klass.table_name) || # has_many or has_one
+                      (assoc_klass.table_name.start_with?('public.') && _all_tables.include?(assoc_klass.table_name[7..-1]))
+                  inverse_foreign_keys = is_polymorphic_hm ? [assoc.type, assoc.foreign_key] : [assoc.inverse_of&.foreign_key&.to_s]
+                  missing_key_columns = inverse_foreign_keys - assoc_klass.columns.map(&:name)
+                  if missing_key_columns.empty?
+                    puts "* Missing inverse foreign key for #{this_klass.name} #{belongs_to_or_has_many} :#{assoc.name}" if inverse_foreign_keys.first.nil?
+                    # puts "Has columns #{inverse_foreign_keys.inspect}"
+                    [[inverse_foreign_keys, assoc_klass], assoc_klass]
+                  else
+                    if inverse_foreign_keys.length > 1
+                      puts "* #{assoc_klass.name} model - missing #{missing_key_columns.join(' and ')} columns to allow it to support polymorphic inheritance."
+                    else
+                      puts "* #{this_klass.name} model - problem with:  \"#{belongs_to_or_has_many} :#{assoc.name}\"."
+
+                      # Most general related parent class in case we're STI
+                      root_class = test_class = this_klass
+                      while (test_class = test_class.superclass) != ActiveRecord::Base
+                        root_class = test_class unless test_class.abstract_class?
+                      end
+                      # If we haven't yet found a match, search for any appropriate unused foreign key that belongs_to the primary class
+                      is_mentioned_consider = false
+                      if (inverses = _find_assocs(:belongs_to, assoc_klass, root_class, errored_assocs, assoc.foreign_key)).empty?
+                        if inverse_foreign_keys.first.nil?
+                          # So we can rule them out, find the belongs_tos that already are inverses of any other relevant has_many
+                          hm_assocs = _find_assocs(:has_many, this_klass, assoc_klass, errored_assocs)
+                          hm_inverses = hm_assocs.each_with_object([]) do |hm, s|
+                            s << hm.inverse_of if hm.inverse_of
+                            s
+                          end
+                          # Remaining belongs_tos are also good candidates to become an inverse_of, so we'll suggest
+                          # both to establish a :foreign_key and also duing the inverses.present? check an :inverse_of.
+                          inverses = _find_assocs(:belongs_to, assoc_klass, root_class, errored_assocs).reject do |bt|
+                            hm_inverses.include?(bt)
+                          end
+                          fks = inverses.map(&:foreign_key)
+                          # All that and still no matches?
+                          unless fks.present? || assoc_klass.columns.map(&:name).include?(suggested_fk = "#{root_class.name.underscore}_id")
+                            # Find any polymorphic association on this model (that we're not already tied to) that could be used.
+                            poly_hms = all_polymorphics.each_with_object([]) do |p, s|
+                              if p.first.first == assoc_klass &&
+                                 p.last.none? { |poly_hm| this_klass <= poly_hm.first } # <= to deal with polymorphic inheritance
+                                s << p.first
+                              end
+                              s
+                            end
+
+                            # Consider all the HMT with through: :contacts, find their source(s)
+                            poly_hmts = this_klass.reflect_on_all_associations.each_with_object([]) do |a, s|
+                              if [:has_many, :has_one].include?(a.macro) && a.options[:source] &&
+                                 a.options[:through] == assoc.name
+                                s << a.options[:source]
+                              end
+                              s
+                            end
+                            poly_hms_hmts = poly_hms.select { |poly_hm| poly_hmts.include?(poly_hm.last) }
+                            poly_hms = poly_hms_hmts unless poly_hms_hmts.blank?
+
+                            poly_hms.map! { |poly_hm| "\"as: :#{poly_hm.last}\"" }
+                            if poly_hms.blank?
+                              puts "    Consider removing this #{belongs_to_or_has_many} because the #{assoc_klass.name} model does not include a column named \"#{suggested_fk}\"."
+                            else
+                              puts "    Consider adding #{poly_hms.join(' or ')} to establish a valid polymorphic association."
+                            end
+                            is_mentioned_consider = true
+                          end
+                          unless fks.empty? || fks.include?(assoc.foreign_key.to_s)
+                            puts "    Consider adding #{fks.map { |fk| "\"foreign_key: :#{fk}\"" }.join(' or ')} (or some other appropriate column from #{assoc_klass.name}) to this #{belongs_to_or_has_many} entry."
+                            is_mentioned_consider = true
+                          end
+                        else
+                          puts "    (Cannot find foreign key \"#{inverse_foreign_keys.first.inspect}\" in #{assoc_klass.name}.)"
+                        end
+                      end
+                      if inverses.empty?
+                        opposite_macro = assoc.belongs_to? ? 'has_many or has_one' : 'belongs_to'
+                        puts "    (Could not identify any inverse #{opposite_macro} association in the #{assoc_klass.name} model.)"
+                      else
+                        print is_mentioned_consider ? '    Also consider ' : '    Consider '
+                        puts "adding \"#{inverses.map { |x| "inverse_of: :#{x.name}" }.join(' or ')}\" to this entry."
+                      end
+                    end
+                    nil
+                  end
+                else
+                  puts "* Missing table #{assoc_klass.table_name} for class #{assoc_klass.name}"
+                  puts '    (Maybe try running:  bin/rails db:migrate )'
+                  nil # Related has_* is missing its table
                 end
-                nil
+              if new_assoc.nil?
+                errored_assocs << assoc
+              else
+                assocs[assoc.name] = new_assoc
               end
             end
-          if new_assoc.nil?
-            errored_assocs << assoc
-          else
-            assocs[assoc.name] = new_assoc
+          end
+
+          # Include all columns except for the primary key, any foreign keys, and excluded_columns
+          # %%% add EXCLUDED_ALL_COLUMNS || ...
+          excluded_columns = %w[created_at updated_at deleted_at]
+          (this_klass.columns.map(&:name) - this_primary_key - this_belongs_tos - excluded_columns).each do |column|
+            template << column.to_sym
+          end
+          # Okay, at this point it really searches for the uniques, and in the "strict" (not loose) kind of way
+          requireds += _find_requireds(this_klass, false, [this_klass.primary_key]).first.map { |r| "#{path}#{r}".to_sym }
+          # Now add the foreign keys and any has_manys in the form of references to associated models
+          assocs.each do |k, assoc|
+            # # assoc.first describes this foreign key and class, and is used for a "reverse poison"
+            # # detection so we don't fold back on ourselves
+            next if poison_links.include?(assoc.first)
+
+            is_has_many = (assoc.first.last == assoc.last)
+            if hops.zero?
+              # For has_one or has_many, exclude with priority the foreign key column(s) we rode in here on
+              priority_excluded_columns = assoc.first.first if is_has_many
+              # puts "Excluded: #{priority_excluded_columns.inspect}"
+              unique, new_requireds = _suggest_unique_column(assoc.last, priority_excluded_columns, "#{path}#{k}_")
+              template << { k => unique }
+              requireds += new_requireds
+            else
+              new_poison_links =
+                if is_has_many
+                  # binding.pry if assoc.first.last.nil?
+                  # has_many is simple, just exclude how we got here from the foreign table
+                  [assoc.first]
+                else
+                  # belongs_to is more involved since there may be multiple foreign keys which point
+                  # from the foreign table to this primary one, so exclude all these links.
+                  _find_assocs(:belongs_to, assoc.first.last, assoc.last, errored_assocs).map do |f_assoc|
+                    [[f_assoc.foreign_key.to_s], f_assoc.active_record]
+                  end
+                end
+              # puts "New Poison: #{new_poison_links.map{|a| "#{a.first.inspect} - #{a.last.name}"}.join(' / ')}"
+              # if (poison_links & new_poison_links).empty?
+              #   Store the ones to do the next round
+              #   puts "Test against #{assoc.first.inspect}"
+              template << { k => (next_template = []) }
+              next_layer << [do_has_many, assoc.last, "#{path}#{k}_", next_template]
+              poison_links += (new_poison_links - poison_links)
+              # end
+            end
           end
         end
+        break if hops.zero? || next_layer.empty?
+
+        hops -= 1
+        this_layer = next_layer
       end
+      (buggy_bts - bad_polymorphic_hms).each do |bad_bt|
+        puts "* #{bad_bt.first.name} model - problem with:  \"belongs_to :#{bad_bt.last.name}\"  because there is no \"#{bad_bt.last.class_name}\" model."
+      end
+      [whole_template, requireds]
+    end
 
-      # Include all columns except for the primary key, any foreign keys, and excluded_columns
-      # %%% add EXCLUDED_ALL_COLUMNS || ...
-      excluded_columns = %w[created_at updated_at deleted_at]
-      template = (this_klass.columns.map(&:name) - this_primary_key - this_belongs_tos - excluded_columns)
-      template.map!(&:to_sym)
-      requireds = _find_requireds(this_klass).map { |r| "#{path}#{r}".to_sym }
-      # Now add the foreign keys and any has_manys in the form of references to associated models
-      assocs.each do |k, assoc|
-        # assoc.first describes this foreign key and class, and is used for a "reverse poison"
-        # detection so we don't fold back on ourselves
-        next if poison_links.include?(assoc.first)
+    # Load all models
+    # %%% Note that this works in Rails 5.x, but may not work in Rails 6.0 and later, which uses the Zeitwerk loader by default:
+    def self._all_models
+      unless ActiveRecord::Base.instance_variable_get(:@eager_loaded_all)
+        if ActiveRecord.version < ::Gem::Version.new('4.0')
+          Rails.configuration.eager_load_paths
+        else
+          Rails.configuration.eager_load_namespaces.select { |ns| ns < Rails::Application }.each(&:eager_load!)
+        end
+        ActiveRecord::Base.instance_variable_set(:@eager_loaded_all, true)
+      end
+      ActiveRecord::Base.descendants
+    end
 
-        is_has_many = (assoc.first.last == assoc.last)
-        # puts "#{k} #{hops}"
-        unique, new_requireds =
-          if hops.zero?
-            # For has_one or has_many, exclude with priority the foreign key column(s) we rode in here on
-            priority_excluded_columns = assoc.first.first if is_has_many
-            # puts "Excluded: #{priority_excluded_columns.inspect}"
-            _suggest_unique_column(assoc.last, priority_excluded_columns, "#{path}#{k}_")
-          else
-            new_poison_links =
-              if is_has_many
-                # has_many is simple, just exclude how we got here from the foreign table
-                [assoc.first]
+    # Load all tables
+    def self._all_tables
+      unless (all_tables = ActiveRecord::Base.instance_variable_get(:@_all_tables))
+        sql = if ActiveRecord::Base.connection.class.name.end_with?('::SQLite3Adapter')
+                "SELECT DISTINCT name AS table_name FROM sqlite_master WHERE type = 'table'"
               else
-                # belongs_to is more involved since there may be multiple foreign keys which point
-                # from the foreign table to this primary one, so exclude all these links.
-                _find_belongs_tos(assoc.first.last, assoc.last, errored_assocs).map do |f_assoc|
-                  [[f_assoc.foreign_key.to_s], f_assoc.active_record]
-                end
+                # For everything else, which would be "::PostgreSQLAdapter", "::MysqlAdapter", or "::Mysql2Adapter":
+                "SELECT DISTINCT table_name FROM INFORMATION_SCHEMA.TABLES WHERE table_type = 'BASE TABLE'"
               end
-            # puts "New Poison: #{new_poison_links.inspect}"
-            _suggest_template(hops - 1, do_has_many, assoc.last, poison_links + new_poison_links, "#{path}#{k}_")
-          end
-        template << { k => unique }
-        requireds += new_requireds
+        # The MySQL version of execute_sql returns arrays instead of a hash when there's just one column asked for.
+        all_tables = ActiveRecord::Base.execute_sql(sql).each_with_object({}) do |row, s|
+          s[row.is_a?(Array) ? row.first : row['table_name']] = nil
+          s
+        end
+        ActiveRecord::Base.instance_variable_set(:@_all_tables, all_tables)
       end
-      [template, requireds]
+      all_tables
     end
 
     # Find belongs_tos for this model to one more more other klasses
-    def self._find_belongs_tos(klass, to_klass, errored_assocs)
-      klass.reflect_on_all_associations.each_with_object([]) do |bt_assoc, s|
-        # .is_a?(ActiveRecord::Reflection::BelongsToReflection)
-        next unless bt_assoc.belongs_to? && !errored_assocs.include?(bt_assoc)
+    def self._find_assocs(macro, klass, to_klass, errored_assocs, using_fk = nil)
+      case macro
+      when :belongs_to
+        klass.reflect_on_all_associations.each_with_object([]) do |bt_assoc, s|
+          next unless bt_assoc.belongs_to? && !errored_assocs.include?(bt_assoc)
+
+          begin
+            s << bt_assoc if !bt_assoc.options[:polymorphic] && bt_assoc.klass <= to_klass &&
+                             (using_fk.nil? || bt_assoc.foreign_key == using_fk)
+          rescue NameError
+            errored_assocs << bt_assoc
+            puts "* #{bt_assoc.active_record.name} model -  \"belongs_to :#{bt_assoc.name}\"  could not find a model named #{bt_assoc.class_name}."
+          end
+          s
+        end
+      when :has_many # Also :has_one
+        klass.reflect_on_all_associations.each_with_object([]) do |hm_assoc, s|
+          next if ![:has_many, :has_one].include?(hm_assoc.macro) || errored_assocs.include?(hm_assoc) ||
+                  (Object.const_defined?('PaperTrail::Version') && hm_assoc.klass <= PaperTrail::Version && hm_assoc.options.include?(:as)) # Skip any PaperTrail associations
 
-        begin
-          s << bt_assoc if !bt_assoc.options[:polymorphic] && bt_assoc.klass == to_klass
-        rescue NameError
-          errored_assocs << bt_assoc
-          puts "* In the #{bt_assoc.active_record.name} model  \"belongs_to :#{bt_assoc.name}\"  could not find a model named #{bt_assoc.class_name}."
+          s << hm_assoc if hm_assoc.klass == to_klass && (using_fk.nil? || hm_assoc.foreign_key == using_fk)
+          s
         end
       end
     end
@@ -197,36 +403,20 @@ module DutyFree
       # ...
       # Not available, so grasping at straws, just search for any available column
       # %%% add EXCLUDED_UNIQUE_COLUMNS || ...
-      klass_columns = klass.columns
-
-      # Requireds takes its cues from all attributes having a presence validator
-      requireds = _find_requireds(klass)
-      klass_columns = klass_columns.reject { |col| priority_excluded_columns.include?(col.name) } if priority_excluded_columns
-      excluded_columns = %w[created_at updated_at deleted_at]
-      unique = [(
-        # Find the first text field of a required if one exists
-        klass_columns.find { |col| requireds.include?(col.name) && col.type == :string }&.name ||
-        # Find the first text field, now of a non-required, if one exists
-        klass_columns.find { |col| col.type == :string }&.name ||
-        # If no string then look for the first non-PK that is also not a foreign key or created_at or updated_at
-        klass_columns.find do |col|
-          requireds.include?(col.name) && col.name != klass.primary_key && !excluded_columns.include?(col.name)
-        end&.name ||
-        # And now the same but not a required, the first non-PK that is also not a foreign key or created_at or updated_at
-        klass_columns.find do |col|
-          col.name != klass.primary_key && !excluded_columns.include?(col.name)
-        end&.name ||
-        # Finally just accept the PK if nothing else
-        klass.primary_key
-      ).to_sym]
-
-      [unique, requireds.map { |r| "#{path}#{r}".to_sym }]
+      uniques, requireds = _find_requireds(klass, true, priority_excluded_columns)
+      [[uniques.first.to_sym], requireds.map { |r| "#{path}#{r}".to_sym }]
     end
 
-    def self._find_requireds(klass)
+    def self._find_requireds(klass, is_loose = false, priority_excluded_columns = nil)
       errored_columns = ::DutyFree.instance_variable_get(:@errored_columns)
-      klass.validators.select do |v|
-        v.is_a?(ActiveRecord::Validations::PresenceValidator)
+      # %%% In case we need to exclude foreign keys in the future, this will do it:
+      # bts = klass.reflect_on_all_associations.each_with_object([]) do |bt_assoc, s|
+      #   next unless bt_assoc.belongs_to?
+
+      #   s << bt_assoc.name
+      # end
+      requireds = klass.validators.select do |v|
+        v.is_a?(ActiveRecord::Validations::PresenceValidator) # && (v.attributes & bts).empty?
       end.each_with_object([]) do |v, s|
         v.attributes.each do |a|
           attrib = a.to_s
@@ -236,17 +426,39 @@ module DutyFree
           if klass.columns.map(&:name).include?(attrib)
             s << attrib
           else
-            hm_and_bt_names = klass.reflect_on_all_associations.each_with_object([]) do |assoc, names|
-              names << assoc.name.to_s if [:belongs_to, :has_many, :has_one].include?(assoc.macro)
-              names
-            end
-            unless hm_and_bt_names.include?(attrib)
-              puts "* In the #{klass.name} model  \"validates_presence_of :#{attrib}\"  should be removed as it does not refer to any existing column."
+            unless klass.instance_methods.map(&:to_s).include?(attrib)
+              puts "* #{klass.name} model -  \"validates_presence_of :#{attrib}\"  should be removed as it does not refer to any existing column or relation."
               errored_columns << klass_col
             end
           end
         end
       end
+      klass_columns = klass.columns
+
+      # Take our cues from all attributes having a presence validator
+      klass_columns = klass_columns.reject { |col| priority_excluded_columns.include?(col.name) } if priority_excluded_columns
+      excluded_columns = %w[created_at updated_at deleted_at]
+
+      # First find any text fields that are required
+      uniques = klass_columns.select { |col| requireds.include?(col.name) && [:string, :text].include?(col.type) }
+      # If not that then find any text field, even those not required
+      uniques = klass_columns.select { |col| [:string, :text].include?(col.type) } if is_loose && uniques.empty?
+      # If still not then look for any required non-PK that is also not a foreign key or created_at or updated_at
+      if uniques.empty?
+        uniques = klass_columns.select do |col|
+          requireds.include?(col.name) && col.name != klass.primary_key && !excluded_columns.include?(col.name)
+        end
+      end
+      # If still nothing then the same but not a required, any non-PK that is also not a foreign key or created_at or updated_at
+      if is_loose && uniques.empty?
+        uniques = klass_columns.select do |col|
+          col.name != klass.primary_key && !excluded_columns.include?(col.name)
+        end
+      end
+      uniques.map!(&:name)
+      # Finally if nothing else then just accept the PK, if there is one
+      uniques = [klass.primary_key] if klass.primary_key && uniques.empty? && (!priority_excluded_columns || priority_excluded_columns.exclude?(klass.primary_key))
+      [uniques, requireds]
     end
 
     # Show a "pretty" version of IMPORT_TEMPLATE, to be placed in a model

data/lib/duty_free/version_number.rb

@@ -5,7 +5,7 @@ module DutyFree
   module VERSION
     MAJOR = 1
     MINOR = 0
-    TINY = 8
+    TINY = 9
 
     # PRE is nil unless it's a pre-release (beta, RC, etc.)
     PRE = nil

data/lib/duty_free.rb

@@ -129,9 +129,21 @@ end
 # Major compatibility fixes for ActiveRecord < 4.2
 # ================================================
 ActiveSupport.on_load(:active_record) do
-  # Rails < 4.0 cannot do #find_by, #find_or_create_by, or do #pluck on multiple columns, so here are the patches:
-  if ActiveRecord.version < ::Gem::Version.new('4.0')
-    module ActiveRecord
+  # rubocop:disable Lint/ConstantDefinitionInBlock
+  module ActiveRecord
+    class Base
+      unless respond_to?(:execute_sql)
+        class << self
+          def execute_sql(sql, *param_array)
+            param_array = param_array.first if param_array.length == 1 && param_array.first.is_a?(Array)
+            connection.execute(send(:sanitize_sql_array, [sql] + param_array))
+          end
+        end
+      end
+    end
+
+    # Rails < 4.0 cannot do #find_by, #find_or_create_by, or do #pluck on multiple columns, so here are the patches:
+    if version < ::Gem::Version.new('4.0')
       # Normally find_by is in FinderMethods, which older AR doesn't have
       module Calculations
         def find_by(*args)
@@ -280,6 +292,7 @@ ActiveSupport.on_load(:active_record) do
       end
     end
   end
+  # rubocop:enable Lint/ConstantDefinitionInBlock
 
   # Rails < 4.2 is not innately compatible with Ruby 2.4 and later, and comes up with:
   # "TypeError: Cannot visit Integer" unless we patch like this:
@@ -322,6 +335,7 @@ ActiveSupport.on_load(:active_record) do
   # First part of arel_table_type stuff:
   # ------------------------------------
   # (more found below)
+  # was:  ActiveRecord.version >= ::Gem::Version.new('3.2') &&
   if ActiveRecord.version < ::Gem::Version.new('5.0')
     # Used by Util#_arel_table_type
     module ActiveRecord
@@ -336,6 +350,17 @@ ActiveSupport.on_load(:active_record) do
   end
 
   include ::DutyFree::Extensions
+
+  unless ::DutyFree::Extensions::IS_AMOEBA
+    # Add amoeba-compatible support
+    module ActiveRecord
+      class Base
+        def self.amoeba(*args)
+          puts "Amoeba called from #{name} with #{args.inspect}"
+        end
+      end
+    end
+  end
 end
 
 # Do this earlier because stuff here gets mixed into JoinDependency::JoinAssociation and AssociationScope

data/lib/generators/duty_free/model_generator.rb

@@ -0,0 +1,349 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+require 'rails/generators/active_record'
+require 'fancy_gets'
+
+module DutyFree
+  # Auto-generates an IMPORT_TEMPLATE entry for a model
+  class ModelGenerator < ::Rails::Generators::Base
+    include FancyGets
+    # include ::Rails::Generators::Migration
+
+    # # source_root File.expand_path('templates', __dir__)
+    # class_option(
+    #   :with_changes,
+    #   type: :boolean,
+    #   default: false,
+    #   desc: 'Add IMPORT_TEMPLATE to model'
+    # )
+
+    desc 'Adds an appropriate IMPORT_TEMPLATE entry into a model of your choosing so that' \
+         '  DutyFree can perform exports, and with the Pro version, the same template also' \
+         '  does imports.'
+
+    def df_model_template
+      # %%% If Apartment is active, ask which schema they want
+
+      # Load all models
+      Rails.configuration.eager_load_namespaces.select { |ns| ns < Rails::Application }.each(&:eager_load!)
+
+      # Generate a list of viable models that can be chosen
+      longest_length = 0
+      model_info = Hash.new { |h, k| h[k] = {} }
+      tableless = Hash.new { |h, k| h[k] = [] }
+      models = ActiveRecord::Base.descendants.reject do |m|
+        trouble = if m.abstract_class?
+                    true
+                  elsif !m.table_exists?
+                    tableless[m.table_name] << m.name
+                    ' (No Table)'
+                  else
+                    this_f_keys = (model_info[m][:f_keys] = m.reflect_on_all_associations.select { |a| a.macro == :belongs_to }) || []
+                    column_names = (model_info[m][:column_names] = m.columns.map(&:name) - [m.primary_key, 'created_at', 'updated_at', 'deleted_at'] - this_f_keys.map(&:foreign_key))
+                    if column_names.empty? && this_f_keys && !this_f_keys.empty?
+                      fk_message = ", although #{this_f_keys.length} foreign keys"
+                      " (No columns#{fk_message})"
+                    end
+                  end
+        # puts "#{m.name}#{trouble}" if trouble&.is_a?(String)
+        trouble
+      end
+      models.sort! do |a, b| # Sort first to separate namespaced stuff from the rest, then alphabetically
+        is_a_namespaced = a.name.include?('::')
+        is_b_namespaced = b.name.include?('::')
+        if is_a_namespaced && !is_b_namespaced
+          1
+        elsif !is_a_namespaced && is_b_namespaced
+          -1
+        else
+          a.name <=> b.name
+        end
+      end
+      models.each do |m| # Find longest name in the list for future use to show lists on the right side of the screen
+        # Strangely this can't be inlined since it assigns to "len"
+        if longest_length < (len = m.name.length)
+          longest_length = len
+        end
+      end
+
+      model_name = ARGV[0]&.camelize
+      unless (starting = models.find { |m| m.name == model_name })
+        puts "#{"Couldn't find #{model_name}.  " if model_name}Pick a model to start from:"
+        starting = gets_list(
+          list: models,
+          on_select: proc do |item|
+            selected = item[:selected] || item[:focused]
+            this_model_info = model_info[selected]
+            selected.name + " (#{(this_model_info[:column_names] + this_model_info[:f_keys].map(&:name).map(&:upcase)).join(', ')})"
+          end
+        )
+      end
+      puts "\nThinking..."
+
+      # %%% Find out how many hops at most we can go from this model
+      max_hm_nav = starting.suggest_template(-1, true, false)
+      max_bt_nav = starting.suggest_template(-1, false, false)
+      hops_with_hm, num_hm_hops_tables = calc_num_hops([[starting, max_hm_nav[:all]]], models)
+      hops, num_hops_tables = calc_num_hops([[starting, max_bt_nav[:all]]], models)
+      # print "\b" * 11
+      unless hops_with_hm.length == hops.length
+        starting_name = starting.name
+        unless (is_hm = ARGV[1]&.downcase)
+          puts "Navigate from #{starting_name} using:\n#{'=' * (21 + starting_name.length)}"
+          is_hm = gets_list(
+            ["Only belongs_to (max of #{hops.length} hops and #{num_hops_tables} tables)",
+             "has_many as well as belongs_to (max of #{hops_with_hm.length} hops and #{num_hm_hops_tables} tables)"]
+          )
+        end
+        is_hm = is_hm.start_with?('has_many') || is_hm[0] == 'y'
+        hops = hops_with_hm if is_hm
+      end
+
+      unless (num_hops = ARGV[2]&.to_i)
+        puts "\nNow, how many hops total would you like to navigate?"
+        index = 0
+        cumulative = 0
+        hops_list = ['0'] + hops.map { |h| "#{index += 1} (#{cumulative += h.length} linkages)" }
+        num_hops = gets_list(
+          list: hops_list,
+          on_select: proc do |value|
+                       associations = Hash.new { |h, k| h[k] = 0 }
+                       index = (value[:selected] || value[:focused]).split(' ').first.to_i - 1
+                       layer = hops[index] if index >= 0
+                       layer ||= []
+                       layer.each { |i| associations[i.last] += 1 }
+                       associations.each { |k, v| associations.delete(k) if v == 1 }
+                       layer.map do |l|
+                         associations.keys.include?(l.last) ? "#{l.first.name.demodulize} #{l.last}" : l.last
+                       end.join(', ')
+                       # y = model_info[data[:focused].name]
+                       # data[:focused].name + " (#{(y[:column_names] + y[:f_keys].map(&:name).map(&:upcase)).join(', ')})"
+                       # layer.inspect
+                     end
+        ).split(' ').first.to_i
+      end
+
+      print "Navigating from #{starting_name}" if model_name
+      puts "\nOkay, #{num_hops} hops#{', including has_many,' if is_hm} it is!"
+      # Grab the console output from this:
+      original_stdout = $stdout
+      $stdout = StringIO.new
+      starting.suggest_template(num_hops, is_hm)
+      output = $stdout
+      $stdout = original_stdout
+      filename = nil
+      output.rewind
+      lines = output.each_line.each_with_object([]) do |line, s|
+        if line == "\n"
+          # Do nothing
+        elsif filename
+          s << line
+        elsif line.start_with?('# Place the following into ')
+          filename = line[27..-1]&.split(':')&.first
+        end
+        s
+      end
+
+      model_file = File.open(filename, 'r+')
+      insert_at = nil
+      starting_name = starting.name.demodulize
+      loop do
+        break if model_file.eof?
+
+        line_parts = model_file.readline.strip.gsub(/ +/, ' ').split(' ')
+        if line_parts.first == 'class' && line_parts[1] && (line_parts[1] == starting_name || line_parts[1].end_with?("::#{starting_name}"))
+          insert_at = model_file.pos
+          break
+        end
+      end
+      line = nil
+      import_template_blocks = []
+      import_template_block = nil
+      indentation = nil
+      # See if there's already any IMPORT_TEMPLATE entries in the model file.
+      # If there already is just one, we will comment it out if needs be before adding a fresh one.
+      loop do
+        break if model_file.eof?
+
+        line_parts = (line = model_file.readline).strip.split(/[\s=]+/)
+        indentation ||= line[0...(/\S/ =~ line)]
+        case line_parts[-2..-1]
+        when ['IMPORT_TEMPLATE', '{']
+          import_template_blocks << import_template_block if import_template_block
+          import_template_block = [model_file.pos - line.length, nil, line.strip[0] == '#', []]
+        when ['#', '------------------------------------------']
+          import_template_block[1] = model_file.pos if import_template_block # && import_template_block[1].nil?
+        end
+        next unless import_template_block
+
+        # Collect all the lines of any existing block
+        import_template_block[3] << line
+        # Cap this one if it's done
+        if import_template_block[1]
+          import_template_blocks << import_template_block
+          import_template_block = nil
+        end
+      end
+      import_template_blocks << import_template_block if import_template_block
+      comments = nil
+      is_add_cr = nil
+      if import_template_blocks.length > 1
+        # %%% maybe in the future:  remove any older commented ones
+        puts 'Found multiple existing import template blocks.  Will not attempt to automatically add yet another.'
+        insert_at = nil
+      elsif import_template_blocks.length == 1
+        # Get set up to add the new block after the existing one
+        insert_at = (import_template_block = import_template_blocks.first)[1]
+        if insert_at.nil?
+          puts "Found what looked like the start of an existing IMPORT_TEMPLATE block, but couldn't determine where it ends.  Will not attempt to automatically add anything."
+        elsif import_template_block[2] # Already commented
+          is_add_cr = true
+        else # Needs to be commented
+          # Find what kind and how much indentation is present from the first commented line
+          indentation = import_template_block[3].first[0...(/\S/ =~ import_template_block[3].first)]
+          comments = import_template_block[3].map { |l| "#{l[0...indentation.length]}# #{l[indentation.length..-1]}" }
+        end
+        # else # Must be no IMPORT_TEMPLATE block yet
+        #   insert_at = model_file.pos
+      end
+      if insert_at.nil?
+        puts "Please edit #{filename} manually and add this code:\n\n#{lines.join}"
+      else
+        is_good = ARGV[3]&.downcase&.start_with?('y')
+        args = [starting_name,
+                is_hm ? 'has_many' : 'no',
+                num_hops]
+        args << 'yes' if is_good
+        args = args.each_with_object(+'') do |v, s|
+          s << " #{v}"
+          s
+        end
+        lines.unshift("# Added #{DateTime.now.strftime('%b %d, %Y %I:%M%P')} by running `bin/rails g duty_free:model#{args}`\n")
+        # add a new one afterwards
+        print is_good ? 'Will' : 'OK to'
+        print "#{" comment #{comments.length} existing lines and" if comments} add #{lines.length} new lines to #{filename}"
+        puts is_good ? '.' : '?'
+        if is_good || gets_list(%w[Yes No]) == 'Yes'
+          # Store rest of file
+          model_file.pos = insert_at
+          rest_of_file = model_file.read
+          if comments
+            model_file.pos = import_template_block[0]
+            model_file.write("#{comments.join}\n")
+            puts "Commented #{comments.length} existing lines"
+          else
+            model_file.pos = insert_at
+          end
+          model_file.write("\n") if is_add_cr
+          model_file.write(lines.map { |l| "#{indentation}#{l}" }.join)
+          model_file.write(rest_of_file)
+        end
+      end
+      model_file.close
+    end
+
+  private
+
+    # def calc_num_hops(all, num = 0)
+    #   max_num = num
+    #   all.each do |item|
+    #     if item.is_a?(Hash)
+    #       item.each do |k, v|
+    #         # puts "#{k} - #{num}"
+    #         this_num = calc_num_hops(item[k], num + 1)
+    #         max_num = this_num if this_num > max_num
+    #       end
+    #     end
+    #   end
+    #   max_num
+    # end
+
+    # Breadth first approach
+    def calc_num_hops(this_layer, models = nil)
+      seen_it = {}
+      layers = []
+      loop do
+        this_keys = []
+        next_layer = []
+        this_layer.each do |grouping|
+          klass = grouping.first
+          # binding.pry #unless klass.is_a?(Class)
+          grouping.last.each do |item|
+            next unless item.is_a?(Hash) && !seen_it.include?([klass, (k, v = item.first).first])
+
+            seen_it[[klass, k]] = nil
+            this_keys << [klass, k]
+            this_klass = klass.reflect_on_association(k)&.klass
+            if this_klass.nil? # Perhaps it's polymorphic
+              polymorphics = klass.reflect_on_all_associations.each_with_object([]) do |r, s|
+                prefix = "#{r.name}_"
+                if r.polymorphic? && k.to_s.start_with?(prefix)
+                  suffix = k.to_s[prefix.length..-1]
+                  possible_klass = models.find { |m| m.name.underscore == suffix }
+                  s << [suffix, possible_klass] if possible_klass
+                end
+                s
+              end
+              # binding.pry if polymorphics.length != 1
+              this_klass = polymorphics.first&.last
+            end
+            next_layer << [this_klass, v.select { |ip| ip.is_a?(Hash) }] if this_klass
+          end
+        end
+        layers << this_keys unless this_keys.empty?
+        break if next_layer.empty?
+
+        this_layer = next_layer
+      end
+      # puts "#{k} - #{num}"
+      [layers, seen_it.keys.map(&:first).uniq.length]
+    end
+
+    # # MySQL 5.6 utf8mb4 limit is 191 chars for keys used in indexes.
+    # def item_type_options
+    #   opt = { null: false }
+    #   opt[:limit] = 191 if mysql?
+    #   ", #{opt}"
+    # end
+
+    # def migration_version
+    #   return unless (major = ActiveRecord::VERSION::MAJOR) >= 5
+
+    #   "[#{major}.#{ActiveRecord::VERSION::MINOR}]"
+    # end
+
+    # # Class names of MySQL adapters.
+    # # - `MysqlAdapter` - Used by gems: `mysql`, `activerecord-jdbcmysql-adapter`.
+    # # - `Mysql2Adapter` - Used by `mysql2` gem.
+    # def mysql?
+    #   [
+    #     'ActiveRecord::ConnectionAdapters::MysqlAdapter',
+    #     'ActiveRecord::ConnectionAdapters::Mysql2Adapter'
+    #   ].freeze.include?(ActiveRecord::Base.connection.class.name)
+    # end
+
+    # # Even modern versions of MySQL still use `latin1` as the default character
+    # # encoding. Many users are not aware of this, and run into trouble when they
+    # # try to use DutyFree in apps that otherwise tend to use UTF-8. Postgres, by
+    # # comparison, uses UTF-8 except in the unusual case where the OS is configured
+    # # with a custom locale.
+    # #
+    # # - https://dev.mysql.com/doc/refman/5.7/en/charset-applications.html
+    # # - http://www.postgresql.org/docs/9.4/static/multibyte.html
+    # #
+    # # Furthermore, MySQL's original implementation of UTF-8 was flawed, and had
+    # # to be fixed later by introducing a new charset, `utf8mb4`.
+    # #
+    # # - https://mathiasbynens.be/notes/mysql-utf8mb4
+    # # - https://dev.mysql.com/doc/refman/5.5/en/charset-unicode-utf8mb4.html
+    # #
+    # def versions_table_options
+    #   if mysql?
+    #     ', { options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci" }'
+    #   else
+    #     ''
+    #   end
+    # end
+  end
+end

data/lib/generators/duty_free/templates/create_versions.rb.erb

@@ -1,5 +1,5 @@
-# This migration creates the `versions` table, the only schema PT requires.
-# All other migrations PT provides are optional.
+# This migration creates the `versions` table, the only schema DF requires.
+# All other migrations DF provides are optional.
 class CreateVersions < ActiveRecord::Migration<%= migration_version %>
 
   # The largest text column available in all supported RDBMS is

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: duty_free
 version: !ruby/object:Gem::Version
-  version: 1.0.8
+  version: 1.0.9
 platform: ruby
 authors:
 - Lorin Thwaits
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-06 00:00:00.000000000 Z
+date: 2023-02-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -17,9 +17,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '3.0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '6.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -27,9 +24,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '3.0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '6.2'
 - !ruby/object:Gem::Dependency
   name: appraisal
   requirement: !ruby/object:Gem::Requirement
@@ -214,6 +208,7 @@ files:
 - lib/duty_free/version_number.rb
 - lib/generators/duty_free/USAGE
 - lib/generators/duty_free/install_generator.rb
+- lib/generators/duty_free/model_generator.rb
 - lib/generators/duty_free/templates/add_object_changes_to_versions.rb.erb
 - lib/generators/duty_free/templates/create_versions.rb.erb
 homepage: https://github.com/lorint/duty_free
@@ -235,7 +230,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.6
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Import and Export Data