datashift 0.15.0 → 0.16.0

data/lib/loaders/excel_loader.rb

@@ -11,7 +11,7 @@
 #             i.e pulls data from each column and sends to object.
 #
 require 'datashift/exceptions'
-
+require 'datashift/exceptions'
 
 module DataShift
 
@@ -21,6 +21,38 @@ module DataShift
 
   module ExcelLoading
 
+    include ExcelBase
+
+    attr_accessor :excel
+
+    # Currently struggling to determine the 'end' of data in a spreadsheet
+    # this reflects if current row had any data at all
+    attr_reader :contains_data
+
+    def start_excel( file_name, options = {} )
+
+      @excel = Excel.new
+
+      excel.open(file_name)
+
+      puts "\n\n\nLoading from Excel file: #{file_name}"
+      logger.info("\nStarting Load from Excel file: #{file_name}")
+
+      sheet_number = options[:sheet_number] || 0
+
+      @sheet = excel.worksheet( sheet_number )
+
+      parse_headers(@sheet, options[:header_row])
+
+      raise MissingHeadersError, "No headers found - Check Sheet #{@sheet} is complete and Row #{header_row_index} contains headers" if(excel_headers.empty?)
+
+      # Create a method_mapper which maps list of headers into suitable calls on the Active Record class
+      # For example if model has an attribute 'price' will map columns called Price or price or PRICE etc to this attribute
+      populate_method_mapper_from_headers(excel_headers, options )
+
+      reporter.reset
+    end
+
     #  Options:
     #   [:dummy]           : Perform a dummy run - attempt to load everything but then roll back
     #  
@@ -37,173 +69,155 @@ module DataShift
     def perform_excel_load( file_name, options = {} )
 
       raise MissingHeadersError, "Minimum row for Headers is 0 - passed #{options[:header_row]}" if(options[:header_row] && options[:header_row].to_i < 0)
-           
-      @excel = Excel.new
 
-      @excel.open(file_name)
-        
-      puts "\n\n\nLoading from Excel file: #{file_name}"
+      start_excel(file_name, options)
 
-      sheet_number = options[:sheet_number] || 0
-      
-      @sheet = @excel.worksheet( sheet_number )
+      begin
+        puts "Dummy Run - Changes will be rolled back" if options[:dummy]
 
-      header_row_index =  options[:header_row] || 0
-      @header_row = @sheet.row(header_row_index)
+        load_object_class.transaction do
 
-      raise MissingHeadersError, "No headers found - Check Sheet #{@sheet} is complete and Row #{header_row_index} contains headers" unless(@header_row)
+          @sheet.each_with_index do |row, i|
 
-      @headers = []
+            current_row_idx = i
+            @current_row = row
 
-      # TODO - make more robust - currently end on first empty column
-      # There is no actual max columns in Excel .. you will run out of memory though at some point
-      (0..1024).each do |column|
-        cell = @header_row[column]
-        break unless cell
-        header = "#{cell.to_s}".strip
-        break if header.empty?
-        @headers << header
-      end
+            next if(current_row_idx == header_row_index)
 
-      raise MissingHeadersError, "No headers found - Check Sheet #{@sheet} is complete and Row #{header_row_index} contains headers" if(@headers.empty?)
-      
-      # Create a method_mapper which maps list of headers into suitable calls on the Active Record class
-      # For example if model has an attribute 'price' will map columns called Price, price, PRICE etc to this attribute
-      populate_method_mapper_from_headers( @headers, options )
-      
-      # currently pointless num_rows rubbish i.e inaccurate!
-      #logger.info "Excel Loader processing #{@sheet.num_rows} rows"
-      
-      @reporter.reset
-      
-      begin
-        puts "Dummy Run - Changes will be rolled back" if options[:dummy]
-          
-        load_object_class.transaction do
-       
-          @sheet.each_with_index do |row, i|
-            
-            @current_row = row 
-            
-            next if(i == header_row_index)
-          
             # Excel num_rows seems to return all 'visible' rows, which appears to be greater than the actual data rows
             # (TODO - write spec to process .xls with a huge number of rows)
             #
             # This is rubbish but currently manually detect when actual data ends, this isn't very smart but
             # got no better idea than ending once we hit the first completely empty row
-            break if @current_row.nil?
-            
-            logger.info "Processing Row #{i} : #{@current_row}"
-            
-            contains_data = false
-            
+            break if(@current_row.nil? || @current_row.compact.empty?)
+
+            logger.info "Processing Row #{current_row_idx} : #{@current_row}"
+
+            @contains_data = false
+
             begin
-              # First assign any default values for columns not included in parsed_file
-              process_missing_columns_with_defaults
-
-              # TODO - Smart sorting of column processing order ....
-              # Does not currently ensure mandatory columns (for valid?) processed first but model needs saving
-              # before associations can be processed so user should ensure mandatory columns are prior to associations
-
-              # as part of this we also attempt to save early, for example before assigning to
-              # has_and_belongs_to associations which require the load_object has an id for the join table
-         
-              # Iterate over method_details, working on data out of associated Excel column
-              @method_mapper.method_details.each do |method_detail|
-                       
-                next unless method_detail # TODO populate unmapped with a real MethodDetail that is 'null' and create is_nil
-            
-                logger.info "Processing Column #{method_detail.column_index}"
-                
-                value = @current_row[method_detail.column_index]
-
-                contains_data = true unless(value.nil? || value.to_s.empty?)
-              
-                prepare_data(method_detail, value)
-              
-                process()           
-              end
-                     
+
+              process_excel_row(row)
+
+              # This is rubbish but currently have to manually detect when actual data ends,
+              # no other way to detect when we hit the first completely empty row
+              break unless(contains_data == true)
+
             rescue => e
-              @reporter.processed_object_count += 1
-              
-              failure(@current_row, true)
-              
-              if(verbose)
-                puts "Failed to process row [#{i}] (#{@current_row})" 
-                puts e.inspect, e.backtrace
-              end
-              
-              logger.error "Failed to process row [#{i}] (#{@current_row})"
-              logger.error e.backtrace
-              
-              # don't forget to reset the load object 
+              process_excel_failure(e, true)
+
+              # don't forget to reset the load object
               new_load_object
               next
             end
-            
+
             break unless(contains_data == true)
 
             # currently here as we can only identify the end of a speadsheet by first empty row
             @reporter.processed_object_count += 1
-                        
+
             # TODO - make optional -  all or nothing or carry on and dump out the exception list at end
-            
-            unless(save)
-              failure
-              logger.error "Failed to save row [#{@current_row}]"
-              logger.error load_object.errors.inspect if(load_object)
-            else
-              logger.info "Row #{@current_row} succesfully SAVED : ID #{load_object.id}"
-              @reporter.add_loaded_object(@load_object)
-            end
-            
+
+            save_and_report
+
             # don't forget to reset the object or we'll update rather than create
             new_load_object
 
-          end
-          
+          end   # all rows processed
+
           if(options[:dummy])
             puts "Excel loading stage complete - Dummy run so Rolling Back."
             raise ActiveRecord::Rollback # Don't actually create/upload to DB if we are doing dummy run
           end
-          
+
         end   # TRANSACTION N.B ActiveRecord::Rollback does not propagate outside of the containing transaction block
-      
-      rescue => e  
+
+      rescue => e
         puts "ERROR: Excel loading failed : #{e.inspect}"
         raise e
-      ensure     
+      ensure
         report
       end
-     
+
     end
-    
+
+    def process_excel_failure( e, delete_object = true)
+      failure(@current_row, delete_object)
+
+      if(verbose)
+        puts "perform_excel_load failed in row [#{current_row_idx}] #{@current_row} - #{e.message} :"
+        puts e.backtrace
+      end
+
+      logger.error  "perform_excel_load failed in row [#{current_row_idx}] #{@current_row} - #{e.message} :"
+      logger.error e.backtrace.join("\n")
+    end
+
+
     def value_at(row, column)
       @excel[row, column]
     end
-    
+
+    def process_excel_row(row)
+
+      # First assign any default values for columns
+      process_defaults
+
+      # TODO - Smart sorting of column processing order ....
+      # Does not currently ensure mandatory columns (for valid?) processed first but model needs saving
+      # before associations can be processed so user should ensure mandatory columns are prior to associations
+
+      # as part of this we also attempt to save early, for example before assigning to
+      # has_and_belongs_to associations which require the load_object has an id for the join table
+
+      # Iterate over method_details, working on data out of associated Excel column
+      @method_mapper.method_details.each_with_index do |method_detail, i|
+
+        unless method_detail
+          logger.warn("No method_detail found for column (#{i})")
+          next # TODO populate unmapped with a real MethodDetail that is 'null' and create is_nil
+        end
+
+        logger.info "Processing Column #{method_detail.column_index} (#{method_detail.operator})"
+
+        value = row[method_detail.column_index]
+
+        @contains_data = true unless(value.nil? || value.to_s.empty?)
+
+        process(method_detail, value)
+      end
+
+    end
+
   end
 
 
   class ExcelLoader < LoaderBase
 
     include ExcelLoading
-  
-    def initialize(klass, find_operators = true, object = nil, options = {})
-      super( klass, find_operators, object, options )
+
+    # Setup loading
+    #
+    # Options to drive building the method dictionary for a class, enabling headers to be mapped to operators on that class.
+    #
+    # Options
+    #  :reload           : Force load of the method dictionary for object_class even if already loaded
+    #  :instance_methods : Include setter/delegate style instance methods for assignment, as well as AR columns
+    #  :verbose          : Verbose logging and to STDOUT
+    #
+    def initialize(klass, object = nil, options = {})
+      super( klass, object, options )
       raise "Cannot load - failed to create a #{klass}" unless @load_object
     end
 
 
     def perform_load( file_name, options = {} )
-      
+
       logger.info "Starting bulk load from Excel : #{file_name}"
-       
+
       perform_excel_load( file_name, options )
 
-      puts "Excel loading stage complete - #{loaded_count} rows added."  
+      puts "Excel loading stage complete - #{loaded_count} rows added."
     end
 
   end

data/lib/loaders/loader_base.rb

@@ -19,70 +19,74 @@ module DataShift
 
     include DataShift::Logging
     include DataShift::Querying
-      
+
     attr_reader :headers
 
     attr_accessor :method_mapper
 
+    # The inbound row/line number
+    attr_accessor :current_row_idx
+
     attr_accessor :load_object_class, :load_object
 
     attr_accessor :reporter
     attr_accessor :populator
-    
+
     attr_accessor :config, :verbose
 
+
     def options() return @config; end
-    
+
 
     # Setup loading
     # 
     # Options to drive building the method dictionary for a class, enabling headers to be mapped to operators on that class.
-    #  
-    # find_operators [default = true] : Populate method dictionary with operators and method details
     #      
     # Options
-    #  
     #  :reload           : Force load of the method dictionary for object_class even if already loaded
     #  :instance_methods : Include setter/delegate style instance methods for assignment, as well as AR columns
-    #  :verbose          : Verboise logging and to STDOUT
+    #  :verbose          : Verbose logging and to STDOUT
     #
-    def initialize(object_class, find_operators = true, object = nil, options = {})
+    def initialize(object_class, object = nil, options = {})
       @load_object_class = object_class
-      
+
+      logger.info("Loading objects of type #{@load_object_class} (#{object}")
+
       @populator = if(options[:populator].is_a?(String))
-        ::Object.const_get(options[:populator]).new
-      elsif(options[:populator].is_a?(Class))
-        options[:populator].new
-      else
-        DataShift::Populator.new
-      end
-          
+                     ::Object.const_get(options[:populator]).new
+                   elsif(options[:populator].is_a?(Class))
+                     options[:populator].new
+                   else
+                     DataShift::Populator.new
+                   end
+
       # Gather names of all possible 'setter' methods on AR class (instance variables and associations)
-      if((find_operators && !MethodDictionary::for?(object_class)) || options[:reload])
+      if( !MethodDictionary::for?(object_class) || options[:reload] )
         #puts "DEBUG Building Method Dictionary for class #{object_class}"
-        
+
         meth_dict_opts = options.extract!(:reload, :instance_methods)
         DataShift::MethodDictionary.find_operators( @load_object_class, meth_dict_opts)
-        
+
         # Create dictionary of data on all possible 'setter' methods which can be used to
         # populate or integrate an object of type @load_object_class
         DataShift::MethodDictionary.build_method_details(@load_object_class)
       end
-      
+
       @method_mapper = DataShift::MethodMapper.new
       @config = options.dup    # clone can cause issues like 'can't modify frozen hash'
 
       @verbose = @config[:verbose]
-      
-      puts "Verbose Mode" if(verbose)
+
+      @current_row_idx = 0
+
       @headers = []
-     
+
       @reporter = DataShift::Reporter.new
-      
+
       reset(object)
     end
 
-    
+
     # Based on filename call appropriate loading function
     # Currently supports :
     #   Excel/Open Office files saved as .xls
@@ -103,11 +107,11 @@ module DataShift
     def perform_load( file_name, options = {} )
 
       raise DataShift::BadFile, "Cannot load #{file_name} file not found." unless(File.exists?(file_name))
-        
+
       logger.info("Perform Load Options:\n#{options.inspect}")
-      
+
       ext = File.extname(file_name)
-      
+
       # TODO - make more modular - these methods doing too much, for example move the object creation/reset
       # out of these perform... methods to make it easier to over ride that behaviour
       if(ext.casecmp('.xls') == 0)
@@ -120,9 +124,9 @@ module DataShift
     end
 
     def report
-      @reporter.report 
+      @reporter.report
     end
-    
+
     # Core API
     # 
     # Given a list of free text column names from a file, 
@@ -148,21 +152,21 @@ module DataShift
     #
     def populate_method_mapper_from_headers( headers, options = {} )
       @headers = headers
-      
+
       mandatory = options[:mandatory] || []
-               
+
       strict = (options[:strict] == true)
-      
-      begin 
+
+      begin
         @method_mapper.map_inbound_headers_to_methods( load_object_class, @headers, options )
       rescue => e
         puts e.inspect, e.backtrace
         logger.error("Failed to map header row to set of database operators : #{e.inspect}")
         raise MappingDefinitionError, "Failed to map header row to set of database operators"
       end
-      
+
       unless(@method_mapper.missing_methods.empty?)
-        puts "WARNING: These headings couldn't be mapped to class #{load_object_class} :\n#{@method_mapper.missing_methods.inspect}"
+        logger.warn("Following headings couldn't be mapped to #{load_object_class} \n#{@method_mapper.missing_methods.inspect}")
         raise MappingDefinitionError, "Missing mappings for columns : #{@method_mapper.missing_methods.join(",")}" if(strict)
       end
 
@@ -170,44 +174,55 @@ module DataShift
         @method_mapper.missing_mandatory(mandatory).each { |er| puts "ERROR: Mandatory column missing - expected column '#{er}'" }
         raise MissingMandatoryError, "Mandatory columns missing  - please fix and retry."
       end
-      
+
       @method_mapper
     end
 
 
-    # Process any defaults user has specified, for those columns that are not included in
-    # the incoming import format
-    def process_missing_columns_with_defaults()
-      inbound_ops = @method_mapper.operator_names
-      @populator.default_values.each do |dn, dv|     
-        logger.debug "Processing default value #{dn} : #{dv}"
-        @populator.assignment(dn, @load_object, dv) unless(inbound_ops.include?(dn))
+    #TODO - Move code into Populator
+    # Process columns with a default value specified
+    def process_defaults()
+
+      @populator.default_values.each do |dname, dv|
+
+        method_detail = MethodDictionary.find_method_detail( load_object_class, dname )
+
+        if(method_detail)
+          logger.debug "Applying default value [#{dname}] on (#{method_detail.operator})"
+          @populator.prepare_and_assign(method_detail, load_object, dv)
+        else
+          logger.warn "No operator found for default [#{dname}] trying basic assignment"
+          begin
+            @populator.insistent_assignment(load_object, dv, dname)
+          rescue
+            logger.error "Badly specified default - could not set #{dname}(#{dv})"
+          end
+        end
       end
     end
-    
+
     # Core API - Given a single free text column name from a file, search method mapper for
     # associated operator on base object class.
     # 
     # If suitable association found, process row data and then assign to current load_object
     def find_and_process(column_name, data)
-      
+
       puts "WARNING: MethodDictionary empty for class #{load_object_class}" unless(MethodDictionary.for?(load_object_class))
-        
+
       method_detail = MethodDictionary.find_method_detail( load_object_class, column_name )
 
       if(method_detail)
-        prepare_data(method_detail, data)
-        process()
+        process(method_detail, data)
       else
         puts "No matching method found for column #{column_name}"
         @load_object.errors.add(:base, "No matching method found for column #{column_name}")
       end
     end
-    
-    
+
+
     # Any Config under key 'LoaderBase' is merged over existing options - taking precedence.
     #  
-    # Any Config under a key equal to the full name of the Loader class (e.g DataShift::SpreeHelper::ImageLoader)
+    # Any Config under a key equal to the full name of the Loader class (e.g DataShift::SpreeEcom::ImageLoader)
     # is merged over existing options - taking precedence.
     # 
     #  Format :
@@ -217,161 +232,190 @@ module DataShift
     #
     def configure_from(yaml_file)
 
-      data = YAML::load( File.open(yaml_file) )
-      
-      logger.info("Read Datashift loading config: #{data.inspect}")
-        
+      logger.info("Reading Datashift loader config from: #{yaml_file.inspect}")
+
+      data = YAML::load( ERB.new( IO.read(yaml_file) ).result )
+
+      logger.info("Read Datashift config: #{data.inspect}")
+
       if(data['LoaderBase'])
         @config.merge!(data['LoaderBase'])
       end
-       
-      if(data[self.class.name])    
+
+      if(data[self.class.name])
         @config.merge!(data[self.class.name])
       end
-      
+
       @populator.configure_from(load_object_class, yaml_file)
       logger.info("Loader Options : #{@config.inspect}")
     end
-    
-    # Set member variables to hold details and value.
-    # 
-    # Check supplied value, validate it, and if required :
-    #   set to provided default value
-    #   prepend any provided prefixes 
-    #   add any provided postfixes
-    def prepare_data(method_detail, value)
-      return @populator.prepare_data(method_detail, value)
-    end
-    
-    # Return the find_by operator and the rest of the (row,columns) data
+
+
+    # Return the find_by (where) operator, if specified, otherwise use the heading operator.
+    # i.e where operator embedded in row ,takes precedence over operator in column heading
+    #
+    # Treat rest of the node as the value to use in the where clause e.g
     #   price:0.99
-    # 
-    # Column headings can already contain the operator so possible that row only contains
+    #
+    # Column headings will be used, if the row only contains data e.g
     #   0.99
+    #   
     # We leave it to caller to manage any other aspects or problems in 'rest'
     #
-    def get_find_operator_and_rest(inbound_data)
-        
-      operator, rest = inbound_data.split(Delimiters::name_value_delim) 
-     
-      #puts "DEBUG inbound_data: #{inbound_data} => #{operator} , #{rest}"
-       
+    def get_operator_and_data(inbound_data)
+
+      where_operator, data = inbound_data.split(Delimiters::name_value_delim)
+
+      md = @populator.current_method_detail
+
       # Find by operator embedded in row takes precedence over operator in column heading
-      if(@populator.current_method_detail.find_by_operator)
-        # row contains 0.99 so rest is effectively operator, and operator is in method details
-        if(rest.nil?)
-          rest = operator
-          operator = @populator.current_method_detail.find_by_operator
+      if((data.nil? || data.empty?) && md.find_by_operator)
+        if((where_operator.nil? || where_operator.empty?))  #colum completely empty - check for defaults
+          if(md.find_by_value)
+            data = md.find_by_value
+          else
+            data = Populator::header_default_data(md.operator)
+          end
+        else
+          data = where_operator
         end
+
+        # row contains single entry only so take operator from header via method details
+        where_operator = md.find_by_operator
       end
-       
-      #puts "DEBUG: get_find_operator_and_rest: #{operator} => #{rest}"
-      
-      return operator, rest
+
+      logger.debug("LoaderBase - get_operator_and_data - [#{where_operator}] - [#{data}]")
+
+      return where_operator, data
     end
-    
+
     # Process a value string from a column.
     # Assigning value(s) to correct association on @load_object.
     # Method detail represents a column from a file and it's correlated AR associations.
     # Value string which may contain multiple values for a collection association.
     #
-    def process()  
-      
-      current_method_detail = @populator.current_method_detail
-      current_value         = @populator.current_value
-        
-      logger.info("Current value to assign : #{current_value}")
-      
+    def process(method_detail, value)
+
+      current_method_detail = method_detail
+
+      current_value, current_attribute_hash = @populator.prepare_data(method_detail, value)
+
+      # TODO - Move ALL of this into Populator properly
       if(current_method_detail.operator_for(:has_many))
 
         if(current_method_detail.operator_class && current_value)
 
           # there are times when we need to save early, for example before assigning to
           # has_and_belongs_to associations which require the load_object has an id for the join table
-        
+
           save_if_new
 
           # A single column can contain multiple associations delimited by special char
           # Size:large|Colour:red,green,blue => ['Size:large', 'Colour:red,green,blue']
           columns = current_value.to_s.split( Delimiters::multi_assoc_delim )
 
-          # Size:large|Colour:red,green,blue   => generates find_by_size( 'large' ) and find_all_by_colour( ['red','green','blue'] )
+          # Size:large|Colour:red,green,blue  =>
+          #   find_by_size( 'large' )
+          #   find_all_by_colour( ['red','green','blue'] )
 
           columns.each do |col_str|
-            
-            find_operator, col_values = get_find_operator_and_rest( col_str )
-                      
+
+            find_operator, col_values = get_operator_and_data( col_str )
+
             raise "Cannot perform DB find by #{find_operator}. Expected format key:value" unless(find_operator && col_values)
-             
+
             find_by_values = col_values.split(Delimiters::multi_value_delim)
-            
+
             find_by_values << current_method_detail.find_by_value if(current_method_detail.find_by_value)
-                     
-            if(find_by_values.size > 1)
 
-              #RAILS 4 current_value = current_method_detail.operator_class.send("find_all_by_#{find_operator}", find_by_values )
-              current_value = current_method_detail.operator_class.where(find_operator => find_by_values)
+            found_values = []
 
-              unless(find_by_values.size == current_value.size)
-                found = current_value.collect {|f| f.send(find_operator) }
-                @load_object.errors.add( current_method_detail.operator, "Association with key(s) #{(find_by_values - found).inspect} NOT found")
-                puts "WARNING: Association #{current_method_detail.operator} with key(s) #{(find_by_values - found).inspect} NOT found - Not added."
-                next if(@current_value.empty?)
-              end
+            #if(find_by_values.size() == 1)
+            # logger.info("Find or create #{current_method_detail.operator_class} with #{find_operator} = #{find_by_values.inspect}")
+            #  item = current_method_detail.operator_class.where(find_operator => find_by_values.first).first_or_create
+            #else
+            #  logger.info("Find #{current_method_detail.operator_class} with #{find_operator} = values #{find_by_values.inspect}")
+            #  current_method_detail.operator_class.where(find_operator => find_by_values).all
+            #end
 
-            else
+            operator_class = current_method_detail.operator_class
 
-              current_value = current_method_detail.operator_class.send("find_by_#{find_operator}", find_by_values )
+            logger.info("Find #{current_method_detail.operator_class} with #{find_operator} = #{find_by_values.inspect}")
 
-              unless(current_value)
-                @load_object.errors.add( current_method_detail.operator, "Association with key #{find_by_values} NOT found")
-                puts "WARNING: Association with key #{find_by_values} NOT found - Not added."
-                next
+            find_by_values.each do |v|
+              begin
+                found_values << operator_class.where(find_operator => v).first_or_create
+              rescue => e
+                logger.error(e.inspect)
+                # TODO some way to define if this is a fatal error or not ?
               end
+            end
+
+            logger.info("Scan result #{found_values.inspect}")
 
+            unless(find_by_values.size == found_values.size)
+              found = found_values.collect {|f| f.send(find_operator) }
+              @load_object.errors.add( current_method_detail.operator, "Association with key(s) #{(find_by_values - found).inspect} NOT found")
+              logger.error "Association [#{current_method_detail.operator}] with key(s) #{(find_by_values - found).inspect} NOT found - Not added."
+              next if(found_values.empty?)
             end
 
+            logger.info("Assigning #{found_values.inspect} (#{found_values.class})")
+
             # Lookup Assoc's Model done, now add the found value(s) to load model's collection
-            @populator.assign(current_method_detail, @load_object, current_value)
-          end
+            @populator.prepare_and_assign(current_method_detail, @load_object, found_values)
+          end # END HAS_MANY
         end
-        # END HAS_MANY
       else
         # Nice n simple straight assignment to a column variable
         #puts "INFO: LOADER BASE processing #{method_detail.name}"
-        @populator.assign(current_method_detail, @load_object, current_value)
+        @populator.assign(load_object)
       end
     end
-    
-    
+
+
     # Loading failed. Store a failed object and if requested roll back (destroy) the current load object
     # For use case where object saved early but subsequent required columns fail to process
     # so the load object is invalid
-    
+
     def failure( object = @load_object, rollback = false)
       if(object)
         @reporter.add_failed_object(object)
-      
-        object.destroy if(rollback && object.respond_to?('destroy') && !object.new_record?)
-        
-        new_load_object # don't forget to reset the load object 
+
+        if(rollback && object.respond_to?('destroy') && !object.new_record?)
+          klass = object.class
+          object.destroy
+          object = klass.new
+        end
+      end
+    end
+
+    def save_and_report
+      unless(save)
+        failure
+        logger.error "Failed to save row (#{current_row_idx}) - [#{@current_row}]"
+        logger.error load_object.errors.inspect if(load_object)
+      else
+        logger.info("Successfully SAVED Object with ID #{load_object.id} for Row #{@current_row}")
+        @reporter.add_loaded_object(@load_object)
+        @reporter.success_inbound_count += 1
       end
     end
 
     def save
       return unless( @load_object )
-      
+
       puts "DEBUG: SAVING #{@load_object.class} : #{@load_object.inspect}" if(verbose)
       begin
         return @load_object.save
       rescue => e
-        failure
-        puts "Error saving #{@load_object.class} : #{e.inspect}"
-        logger.error e.backtrace
-        raise "Error in save whilst processing column #{@current_method_detail.name}" if(@config[:strict])
+        logger.error( "Save Error : #{e.inspect} on #{@load_object.class}")
+        logger.error(e.backtrace)
       end
-    end 
-    
+
+      false
+    end
+
     # Reset the loader, including database object to be populated, and load counts
     #
     def reset(object = nil)
@@ -379,7 +423,7 @@ module DataShift
       @reporter.reset
     end
 
-    
+
     def new_load_object
       @load_object = @load_object_class.new
       @load_object
@@ -408,7 +452,7 @@ module DataShift
     def missing_mandatory_headers( mandatory_list )
       [ [*mandatory_list] - @headers].flatten
     end
-    
+
     def find_or_new( klass, condition_hash = {} )
       @records[klass] = klass.find(:all, :conditions => condition_hash)
       if @records[klass].any?
@@ -419,14 +463,14 @@ module DataShift
     end
 
     protected
-    
+
     # Take current column data and split into each association
     # Supported Syntax :
     #  assoc_find_name:value | assoc2_find_name:value | etc
     def get_each_assoc
-      @populator.current_value.to_s.split( Delimiters::multi_assoc_delim )
+      current_value = @populator.current_value.to_s.split( Delimiters::multi_assoc_delim )
     end
-      
+
     private
 
     # This method usually called during processing to avoid errors with associations like
@@ -436,14 +480,14 @@ module DataShift
     # TODO smart ordering of columns dynamically ourselves rather than relying on incoming data order
     def save_if_new
       return unless(load_object.new_record?)
-      
-      if(load_object.valid?)  
+
+      if(load_object.valid?)
         save
       else
-        puts "Cannot Save - Invalid #{load_object.class} - #{load_object.errors.full_messages}" if(verbose)
+        raise DataShift::SaveError.new("Cannot Save - Invalid #{load_object.class} Record - #{load_object.errors.full_messages}")
       end
     end
-  
+
   end
 
 end