datashift 0.40.3 → 0.40.4

data/lib/datashift/loaders/csv_loader.rb

@@ -77,7 +77,7 @@ module DataShift
 
               begin
                 context.process
-              rescue => x
+              rescue StandardError => x
                 if doc_context.all_or_nothing?
                   logger.error('Complete Row aborted - All or nothing set and Current Column failed.')
                   logger.error(x.backtrace.first.inspect)
@@ -97,8 +97,7 @@ module DataShift
             raise ActiveRecord::Rollback # Don't actually create/upload to DB if we are doing dummy run
           end
         end # TRANSACTION N.B ActiveRecord::Rollback does not propagate outside of the containing transaction block
-
-      rescue => e
+      rescue StandardError => e
         puts "ERROR: CSV loading failed : #{e.inspect}"
         raise e
       ensure

data/lib/datashift/loaders/excel_loader.rb

@@ -66,13 +66,18 @@ module DataShift
 
             @binder.bindings.each do |method_binding|
 
+              # TODO: - how does this get inserted - bind headers ?? ignore if no index
+              # #<DataShift::MethodBinding:0x0000000003e26280 @inbound_column=#<DataShift::InboundData::Column:0x0000000003e26258 @header=#<DataShift::Header:0x0000000003e26190 @source="audio", @presentation="audio">, @index=nil,
+              next if method_binding.index.nil?
+
               unless method_binding.valid?
                 logger.warn("No binding was found for column (#{current_row_idx})")
                 next
               end
 
               # If binding to a column, get the value from the cell (bindings can be to internal methods)
-              value = method_binding.index ? row[method_binding.index] : nil
+              #
+              value = row[method_binding.index]
 
               context = doc_context.create_node_context(method_binding, current_row_idx, value)
 
@@ -82,7 +87,7 @@ module DataShift
 
               begin
                 context.process
-              rescue
+              rescue StandardError
                 if doc_context.all_or_nothing?
                   logger.error('All or nothing set and Current Column failed so complete Row aborted')
                   break
@@ -105,8 +110,7 @@ module DataShift
             raise ActiveRecord::Rollback # Don't actually create/upload to DB if we are doing dummy run
           end
         end # TRANSACTION N.B ActiveRecord::Rollback does not propagate outside of the containing transaction block
-
-      rescue => e
+      rescue StandardError => e
         puts "ERROR: Excel loading failed : #{e.inspect}"
         raise e
       ensure

data/lib/datashift/loaders/failure_data.rb

@@ -31,9 +31,7 @@ module DataShift
 
     def destroy_failed_object
       if load_object
-        if load_object.respond_to?('destroy') && !load_object.new_record?
-          load_object.destroy
-        end
+        load_object.destroy if load_object.respond_to?('destroy') && !load_object.new_record?
       end
     end
 

data/lib/datashift/loaders/loader_base.rb

@@ -89,11 +89,9 @@ module DataShift
 
       logger.info("Binding #{headers.size} inbound headers to #{load_object_class.name}")
 
-      @binder ||= DataShift::Binder.new
-
       begin
         binder.map_inbound_headers(load_object_class, headers)
-      rescue => e
+      rescue StandardError => e
         logger.error("Failed to map header row to set of database operators : #{e.inspect}")
         logger.error( e.backtrace )
         raise MappingDefinitionError, 'Failed to map header row to set of database operators'
@@ -141,14 +139,10 @@ module DataShift
 
       logger.info("Reading Datashift loader config from: #{yaml_file.inspect}")
 
-      data = YAML.load( ERB.new( IO.read(yaml_file) ).result )
+      data = Configuration.parse_yaml(yaml_file)
 
       logger.info("Read Datashift config: #{data.inspect}")
 
-      @config.merge!(data['LoaderBase']) if data['LoaderBase']
-
-      @config.merge!(data[self.class.name]) if data[self.class.name]
-
       @binder ||= DataShift::Binder.new
 
       data_flow_schema = DataShift::DataFlowSchema.new

data/lib/datashift/loaders/loader_factory.rb

@@ -16,6 +16,12 @@ module DataShift
 
     class Factory
 
+      # organizer
+
+      # include Interactor::Organizer
+
+      # organize DetermineLoaderKlass, BuildKlassCatalog, LoadConfig, ApplyConfig
+
       # Based on file_name find appropriate Loader
 
       # Currently supports :

data/lib/datashift/loaders/paperclip/attachment_loader.rb

@@ -80,7 +80,7 @@ module DataShift
           logger.info("Finding matching field/association [#{attach_to_field}] on class [#{attach_to_klass}]")
 
           binder.map_inbound_fields(attach_to_klass, attach_to_field)
-        rescue => e
+        rescue StandardError => e
           logger.error("Failed to map #{attach_to_field} to database operator : #{e.inspect}")
           logger.error( e.backtrace )
           raise MappingDefinitionError, 'Failed to map #{attach_to_field} to database operator'

data/lib/datashift/loaders/paperclip/datashift_paperclip.rb

@@ -40,7 +40,7 @@ module DataShift
 
       file = begin
         File.new(attachment_path, 'rb')
-      rescue => e
+      rescue StandardError => e
         logger.error(e.inspect)
         raise PathError, "ERROR : Failed to read image from #{attachment_path}"
       end
@@ -89,7 +89,7 @@ module DataShift
         logger.info("Create paperclip attachment on Class #{klass} - #{paperclip_attributes}")
 
         @attachment = klass.new(paperclip_attributes)
-      rescue => e
+      rescue StandardError => e
         logger.error(e.backtrace.first)
         raise CreateAttachmentFailed, "Failed [#{e.message}] - Creating Attachment [#{attachment_path}] on #{klass}"
       ensure

data/lib/datashift/loaders/paperclip/image_loading.rb

@@ -54,7 +54,7 @@ module DataShift
     # Set of file extensions ImageMagik can process so default glob
     # we use to find image files within directories
     def self.image_magik_glob
-      @im_glob ||= %w(*.3FR *.AAI *.AI *.ART *.ARW *.AVI *.AVS *.BGR *.BGRA
+      @im_glob ||= %w[*.3FR *.AAI *.AI *.ART *.ARW *.AVI *.AVS *.BGR *.BGRA
                       *.BIE *.BMP *.BMP2 *.BMP3 *.BRF *.CAL *.CALS *.CANVAS
                       *.CIN *.CIP *.CLIP *.CMYK *.CMYKA *.CR2 *.CRW *.CUR *.CUT *.DCM *.DCR *.DCX
                       *.DDS *.DFONT *.DJVU *.DNG *.DOT *.DPS *.DPX
@@ -70,7 +70,7 @@ module DataShift
                       *.RGBA *.RGBO *.RLA *.RLE *.SCR *.SCT *.SFW *.SGI *.SR2 *.SRF
                       *.SUN *.SVG *.SVGZ *.TGA *.TIFF *.TIFF64 *.TILE *.TIM *.TTC *.TTF *.UBRL *.UIL *.UYVY *.VDA *.VICAR
                       *.VID *.VIFF *.VST *.WBMP *.WEBP *.WMF *.WMV *.WMZ *.WPG *.X3F
-                      *.XBM *.XC *.XCF *.XPM *.XPS *.XV *.XWD *.YCbCr *.YCbCrA *.YUV)
+                      *.XBM *.XC *.XCF *.XPM *.XPS *.XV *.XWD *.YCbCr *.YCbCrA *.YUV]
       "{#{@im_glob.join(',')}}"
     end
   end

data/lib/datashift/mapping/data_flow_schema.rb

@@ -56,12 +56,12 @@ module DataShift
     end
 
     # @headers=
-    #<DataShift::Header:0x00000004bc37f8
+    # <DataShift::Header:0x00000004bc37f8
     #  @presentation="status_str",
     #  @source="status_str">],
 
     def headers
-      # TODO fix doc context so it can be created 'empty' i.e without AR klass, and always has empty headers
+      # TODO: fix doc context so it can be created 'empty' i.e without AR klass, and always has empty headers
       @nodes.doc_context.try(:headers) || []
     end
 
@@ -83,11 +83,11 @@ module DataShift
       @nodes = create_node_collections(klass, doc_context: doc_context)
 
       klass_to_model_methods( klass ).each_with_index do |mm, i|
-        @nodes.headers.add(mm.operator)   # for a class, the header names, default to the operators (methods)
+        @nodes.headers.add(mm.operator) # for a class, the header names, default to the operators (methods)
 
-        binding = MethodBinding.new(mm.operator, i, mm)
+        binding = MethodBinding.new(mm.operator, mm, idx: i)
 
-        # TODO - do we really need to pass in the doc context when parent nodes already has it ?
+        # TODO: - do we really need to pass in the doc context when parent nodes already has it ?
         @nodes << DataShift::NodeContext.new(@nodes.doc_context, binding, i, nil)
       end
 
@@ -121,17 +121,19 @@ module DataShift
     #
     # See Config generation or lib/datashift/templates/import_export_config.erb for full syntax
     #
-    def prepare_from_file(file_name, locale_key = 'data_flow_schema')
-      @raw_data = ERB.new(File.read(file_name)).result
+    # Returns  DataShift::NodeCollection
+    #
+    def prepare_from_file(yaml_file, locale_key = 'data_flow_schema')
+      @raw_data = ERB.new(File.read(yaml_file)).result
 
-      yaml = YAML.load(raw_data)
+      yaml = Configuration.parse_yaml(yaml_file)
 
       prepare_from_yaml(yaml, locale_key)
     end
 
     def prepare_from_string(text, locale_key = 'data_flow_schema')
       @raw_data = text
-      yaml = YAML.load(raw_data)
+      yaml = YAML.safe_load(raw_data)
 
       prepare_from_yaml(yaml, locale_key)
     end
@@ -144,6 +146,23 @@ module DataShift
 
       locale_section = yaml[locale_key]
 
+      if(locale_section.key?('Global'))
+        global_nodes = locale_section.delete('Global')
+
+        [*global_nodes].each do |c|
+
+          # TODO: what is c ?   a list or hash ?
+          # if DataShift::Configuration.call.respond_to #{c}=
+          # Set the global value e.g
+          # DataShift::Configuration.call.force_inclusion_of_columns = [:audio]
+        end
+      end
+
+      unless locale_section.keys.present?
+        logger.warn('No class related configuration found in YAML syntax- Nothing to process')
+        return DataShift::NodeCollection.new
+      end
+
       class_name = locale_section.keys.first
 
       klass = MapperUtils.class_from_string_or_raise(class_name)
@@ -189,7 +208,7 @@ module DataShift
 
           section = keyed_node.values.first || {}
 
-          # TODO - layout with heading is verbose for no benefit - defunct, simply node.source, node.presentation
+          # TODO: - layout with heading is verbose for no benefit - defunct, simply node.source, node.presentation
           source = section.fetch('heading', {}).fetch('source', nil)
 
           # Unless a specific source mentioned assume the node is the source
@@ -201,19 +220,26 @@ module DataShift
 
           if(section['operator'])
             # Find the domain model method details
+            # byebug
             model_method = model_method_mgr.search(section['operator'])
 
             unless model_method
               operator_type = section['operator_type'] || :method
 
-              # expect one of ModelMethod.supported_types_enum
+              # TODO: validate type ? guess we expect one of ModelMethod.supported_types_enum
               model_method = model_method_mgr.insert(section['operator'], operator_type)
+              # TODO: - This index could be hard coded by the user in YAML or we try to derive it from the headers
+              # byebug
+              method_binding = MethodBinding.new(source, model_method, idx: section['index'])
             end
-
-            method_binding = InternalMethodBinding.new(model_method)
           end
 
-          method_binding ||= MethodBinding.new(source, i, model_method)
+          # Now ensure we bind source/header(and index) to the method tht performs assignment of inbound datum to the model
+          #
+          # TOFIX - This is a bug waiting to happen right ? i is not coming from the headers
+          # so chances are user hasn't made config indexed as per headers
+          # index could be hard coded by the user in YAML or we try to derive it from the headers via the binder ???
+          method_binding ||= MethodBinding.new(source, model_method, idx: i)
 
           node_context = DataShift::NodeContext.new(@nodes.doc_context, method_binding, i, nil)
 

data/lib/datashift/mapping/mapper_utils.rb

@@ -46,8 +46,8 @@ module DataShift
     def self.class_from_string( str )
 
       MapperUtils.const_get_from_string(str.to_s) # Kernel.const_get(model)
-    rescue
-      return nil
+    rescue StandardError
+      nil
 
     end
 
@@ -58,7 +58,7 @@ module DataShift
       begin
         DataShift::MapperUtils.class_from_string(find_table)
       rescue LoadError
-      rescue
+      rescue StandardError
         nil
       end
     end

data/lib/datashift/model_methods/catalogue.rb

@@ -61,19 +61,17 @@ module DataShift
 
         # Find the has_one associations which can be populated via  Model.has_one_name = OtherArModelObject
         if options[:reload] || has_one[klass].nil?
-          if Module.const_defined?(:Mongoid)
-            has_one[klass] = klass.reflect_on_all_associations(:embeds_one).map { |i| i.name.to_s }
-          else
-            has_one[klass] = klass.reflect_on_all_associations(:has_one).map { |i| i.name.to_s }
-          end
+          has_one[klass] = if Module.const_defined?(:Mongoid)
+                             klass.reflect_on_all_associations(:embeds_one).map { |i| i.name.to_s }
+                           else
+                             klass.reflect_on_all_associations(:has_one).map { |i| i.name.to_s }
+                           end
         end
 
         # Find the model's column associations which can be populated via xxxxxx= value
         # Note, not all reflections return method names in same style so we convert all to
         # the raw form i.e without the '='  for consistency
-        if options[:reload] || assignments[klass].nil?
-          build_assignments( klass, options[:instance_methods] )
-        end
+        build_assignments( klass, options[:instance_methods] ) if options[:reload] || assignments[klass].nil?
       end
 
       def self.clear
@@ -155,7 +153,7 @@ module DataShift
         def build_assignments(klass, include_instance_methods)
           begin
             assignments[klass] = Catalogue.column_names(klass)
-          rescue => x
+          rescue StandardError => x
             raise DataShiftException, "Failed to process column_names for class #{klass} - #{x.message}"
           end
 
@@ -172,11 +170,13 @@ module DataShift
 
           assignments[klass].uniq!
 
-          assignments[klass].each do |assign|
-            column_types[klass] ||= {}
-            column_def = klass.columns.find { |col| col.name == assign }
-            column_types[klass].merge!( assign => column_def) if column_def
-          end unless (Module.const_defined?(:Mongoid))
+          unless Module.const_defined?(:Mongoid)
+            assignments[klass].each do |assign|
+              column_types[klass] ||= {}
+              column_def = klass.columns.find { |col| col.name == assign }
+              column_types[klass].merge!( assign => column_def) if column_def
+            end
+          end
         end
 
         def catalogued

data/lib/datashift/model_methods/model_method.rb

@@ -4,7 +4,7 @@
 # License::   MIT
 #
 # Details::   This class holds info on a single Method callable on a domain Model
-#             By holding information on the Type inbound data can be manipulated
+#             By holding information on the Type, inbound data can be manipulated
 #             into the right format for the style of operator; simple assignment,
 #             appending to an association collection or a method call
 #
@@ -17,7 +17,7 @@ module DataShift
     include DataShift::Logging
 
     def self.association_types_enum
-      @assoc_type_enum ||= [:belongs_to, :has_one, :has_many]
+      @assoc_type_enum ||= %i[belongs_to has_one has_many]
       @assoc_type_enum
     end
 
@@ -45,7 +45,7 @@ module DataShift
 
       # Note : Not all assignments will currently have a column type, for example
       # those that are derived from a delegate_belongs_to
-      keys =  Module.const_defined?(:Mongoid) ? klass.fields.keys : klass.columns.map(&:name)
+      keys = Module.const_defined?(:Mongoid) ? klass.fields.keys : klass.columns.map(&:name)
       @connection_adapter_column = keys.find { |col| col == operator } if connection_adapter_column.nil?
 
       @connection_adapter_column = DataShift::ModelMethods::Catalogue.column_type_for(klass, operator) if connection_adapter_column.nil?
@@ -144,12 +144,11 @@ module DataShift
 
         if result.nil?
           begin
-
             first = klass.to_s.split('::').first
             logger.debug "Trying to find operator class with Parent Namespace #{first}"
 
             result = MapperUtils.const_get_from_string("#{first}::#{operator.classify}")
-          rescue => e
+          rescue StandardError => e
             logger.error("Failed to derive Class for #{operator} (#{@operator_type} - #{e.inspect}")
           end
         end
@@ -159,7 +158,7 @@ module DataShift
       elsif connection_adapter_column
         begin
           Kernel.const_get(connection_adapter_column.type.to_s.classify)
-        rescue
+        rescue StandardError
           nil
         end
       end

data/lib/datashift/model_methods/operator.rb

@@ -18,7 +18,7 @@ module DataShift
     # N.B these are in priority order ie. often prefer to process assignments first, then associations
     #
     def self.supported_types_enum
-      @type_enum ||= [:assignment, :enum, :belongs_to, :has_one, :has_many, :method]
+      @type_enum ||= %i[assignment enum belongs_to has_one has_many method paperclip]
       @type_enum
     end
 

data/lib/datashift/node_context.rb

@@ -44,8 +44,7 @@ module DataShift
 
     def process
       populator.prepare_and_assign(self, doc_context.load_object, data)
-    rescue => x
-
+    rescue StandardError => x
       failed = FailureData.new( doc_context.load_object, self, x.message)
 
       failed.error_messages <<  "Failed to process node : #{method_binding.pp}"
@@ -61,7 +60,7 @@ module DataShift
   class EmptyContext < NodeContext
 
     def initialize
-      super(NilClass, DataShift::NoMethodBinding.new, -1, [])
+      super(NilClass, DataShift::NoMethodBinding.new, [], idx: -1)
     end
   end
 

data/lib/datashift/populators/has_many.rb

@@ -80,7 +80,7 @@ module DataShift
           find_by_values.each do |v|
             begin
               found_values << klass.where(field => v).first_or_create
-            rescue => e
+            rescue StandardError => e
               logger.error(e.inspect)
               logger.error("Failed to find or create #{klass} where #{field} => #{v}")
               # TODO: some way to define if this is a fatal error or not ?
@@ -100,7 +100,7 @@ module DataShift
 
           begin
             load_object.send(operator) << found_values
-          rescue => e
+          rescue StandardError => e
             logger.error e.inspect
             logger.error "Cannot assign #{found_values.inspect} to has_many [#{operator}] "
           end

data/lib/datashift/populators/insistent_assignment.rb

@@ -19,13 +19,13 @@ module DataShift
       extend DataShift::Logging
 
       def self.insistent_method_list
-        @insistent_method_list ||= [:to_s, :downcase, :to_i, :to_f, :to_b]
+        @insistent_method_list ||= %i[to_s downcase to_i to_f to_b]
       end
 
       # When looking up an association, when no field provided, try each of these in turn till a match
       # i.e find_by_name, find_by_title, find_by_id
       def self.insistent_find_by_list
-        @insistent_find_by_list ||= [:name, :title, :id]
+        @insistent_find_by_list ||= %i[name title id]
       end
 
       def self.call(record, value, operator)
@@ -49,7 +49,7 @@ module DataShift
           InsistentAssignment.insistent_method_list.each do |f|
             begin
               return if(attempt(record, value.send(f), method))
-            rescue
+            rescue StandardError
             end
           end
         end
@@ -64,7 +64,7 @@ module DataShift
         def attempt(record, value, operator)
           begin
             record.send(operator, value)
-          rescue
+          rescue StandardError
             logger.debug("Brute forced failed for [#{operator}, #{value}]")
             return false
           end