datashift 0.11.1 → 0.12.0

data/lib/loaders/excel_loader.rb

@@ -22,6 +22,8 @@ module DataShift
   module ExcelLoading
 
     #  Options:
+    #   [:dummy]           : Perform a dummy run - attempt to load everything but then roll back
+    #  
     #   [:sheet_number]    : Default is 0. The index of the Excel Worksheet to use.
     #   [:header_row]      : Default is 0. Use alternative row as header definition.
     #   
@@ -72,77 +74,108 @@ module DataShift
       
       logger.info "Excel Loader processing #{@sheet.num_rows} rows"
       
-      loaded_objects.clear
-            
-      load_object_class.transaction do
+      @reporter.reset
+      
+      begin
+        puts "Dummy Run - Changes will be rolled back" if options[:dummy]
+          
+        load_object_class.transaction do
        
-        @sheet.each_with_index do |row, i|
-                 
-          next if(i == header_row_index)
+          @sheet.each_with_index do |row, i|
+            
+            @current_row = row 
+            
+            next if(i == header_row_index)
           
-          # Excel num_rows seems to return all 'visible' rows, which appears to be greater than the actual data rows
-          # (TODO - write spec to process .xls with a huge number of rows)
-          #
-          # This is rubbish but currently manually detect when actual data ends, this isn't very smart but
-          # got no better idea than ending once we hit the first completely empty row
-          break if row.nil?
-
-          contains_data = false
+            # Excel num_rows seems to return all 'visible' rows, which appears to be greater than the actual data rows
+            # (TODO - write spec to process .xls with a huge number of rows)
+            #
+            # This is rubbish but currently manually detect when actual data ends, this isn't very smart but
+            # got no better idea than ending once we hit the first completely empty row
+            break if @current_row.nil?
             
-          # First assign any default values for columns not included in parsed_file
-          process_missing_columns_with_defaults
+            logger.info "Processing Row #{i} : #{@current_row}"
+            
+            contains_data = false
+            
+            begin
+              # First assign any default values for columns not included in parsed_file
+              process_missing_columns_with_defaults
 
-          # TODO - Smart sorting of column processing order ....
-          # Does not currently ensure mandatory columns (for valid?) processed first but model needs saving
-          # before associations can be processed so user should ensure mandatory columns are prior to associations
+              # TODO - Smart sorting of column processing order ....
+              # Does not currently ensure mandatory columns (for valid?) processed first but model needs saving
+              # before associations can be processed so user should ensure mandatory columns are prior to associations
 
-          # as part of this we also attempt to save early, for example before assigning to
-          # has_and_belongs_to associations which require the load_object has an id for the join table
+              # as part of this we also attempt to save early, for example before assigning to
+              # has_and_belongs_to associations which require the load_object has an id for the join table
          
-          # Iterate over method_details, working on data out of associated Excel column
-          @method_mapper.method_details.each do |method_detail|
-                
-            next unless method_detail # TODO populate unmapped with a real MethodDetail that is 'null' and create is_nil
+              # Iterate over method_details, working on data out of associated Excel column
+              @method_mapper.method_details.each do |method_detail|
+                       
+                next unless method_detail # TODO populate unmapped with a real MethodDetail that is 'null' and create is_nil
             
-            value = row[method_detail.column_index]
+                logger.info "Processing Column #{method_detail.column_index}"
+                
+                value = @current_row[method_detail.column_index]
 
-            contains_data = true unless(value.nil? || value.to_s.empty?)
+                contains_data = true unless(value.nil? || value.to_s.empty?)
               
-            prepare_data(method_detail, value)
+                prepare_data(method_detail, value)
               
-            process()
-          end
-                          
-          break unless(contains_data == true)
-
-          # TODO - requirements to handle not valid ?
-          # all or nothing or carry on and dump out the exception list at end
-          #puts "DEBUG: FINAL SAVE #{load_object.inspect}"
-          unless(save)
-            failure
-            logger.error "Failed to save row [#{row}]"
-            logger.error load_object.errors.inspect if(load_object)
-          else
-            logger.info "Row #{row} succesfully SAVED : ID #{load_object.id}"
-          end
+                process()           
+              end
+                     
+            rescue => e
+              @reporter.processed_object_count += 1
+              
+              failure(@current_row, true)
+              logger.error "Failed to process row [#{i}] (#{@current_row})"
+              # don't forget to reset the load object 
+              new_load_object
+              next
+            end
+            
+            break unless(contains_data == true)
+
+            # currently here as we can only identify the end of a speadsheet by first empty row
+            @reporter.processed_object_count += 1
+                        
+            # TODO - make optional -  all or nothing or carry on and dump out the exception list at end
             
-          # don't forget to reset the object or we'll update rather than create
-          new_load_object
+            unless(save)
+              failure
+              logger.error "Failed to save row [#{@current_row}]"
+              logger.error load_object.errors.inspect if(load_object)
+            else
+              logger.info "Row #{@current_row} succesfully SAVED : ID #{load_object.id}"
+               @reporter.add_loaded_object(@load_object)
+            end
+            
+            # don't forget to reset the object or we'll update rather than create
+            new_load_object
 
+          end
+        
+          raise ActiveRecord::Rollback if(options[:dummy]) # Don't actually create/upload to DB if we are doing dummy run
         end
-      end
-      
-      loaded_objects.compact! if(loaded_objects)
       
-      puts "Excel loading stage complete - #{loaded_objects.size} rows added."
-      puts "There were NO failures." if failed_objects.empty?
+      rescue => e
         
-      puts "WARNING : Check logs : #{failed_objects.size} rows contained errors and #{failed_objects.size} records NOT created." unless failed_objects.empty?
+        if e.is_a?(ActiveRecord::Rollback) && options[:dummy]
+          puts "Excel loading stage complete - Dummy run so Rolling Back."
+        else
+          raise e
+        end
+      ensure
+        report
+      end
+     
     end
-
+    
     def value_at(row, column)
       @excel[row, column]
     end
+    
   end
 
 
@@ -159,7 +192,7 @@ module DataShift
     def perform_load( file_name, options = {} )
       perform_excel_load( file_name, options )
 
-      puts "Excel loading stage complete - #{loaded_objects.size} rows added."  
+      puts "Excel loading stage complete - #{loaded_count} rows added."  
     end
 
   end

data/lib/loaders/loader_base.rb

@@ -28,7 +28,7 @@ module DataShift
     attr_accessor :load_object_class, :load_object
     attr_accessor :current_value, :current_method_detail
 
-    attr_accessor :loaded_objects, :failed_objects
+    attr_accessor :reporter
 
     attr_accessor :config, :verbose
 
@@ -51,7 +51,7 @@ module DataShift
       
       # Gather names of all possible 'setter' methods on AR class (instance variables and associations)
       if((find_operators && !MethodDictionary::for?(object_class)) || options[:reload])
-        puts "Building Method Dictionary for class #{object_class}"
+        #puts "DEBUG Building Method Dictionary for class #{object_class}"
         DataShift::MethodDictionary.find_operators( @load_object_class, :reload => options[:reload], :instance_methods => options[:instance_methods] )
         
         # Create dictionary of data on all possible 'setter' methods which can be used to
@@ -73,9 +73,7 @@ module DataShift
       @prefixes       = {}
       @postfixes      = {}
       
-      # TODO - move to own LoadStats or LoadReport class
-      @loaded_objects = []
-      @failed_objects = []
+      @reporter = DataShift::Reporter.new
       
       reset(object)
     end
@@ -88,6 +86,8 @@ module DataShift
     # 
     # OPTIONS :
     #  
+    #  [:dummy]         : Perform a dummy run - attempt to load everything but then roll back
+    #  
     #  strict           : Raise an exception of any headers can't be mapped to an attribute/association
     #  ignore           : List of column headers to ignore when building operator map
     #  mandatory        : List of columns that must be present in headers
@@ -100,8 +100,12 @@ module DataShift
 
       raise DataShift::BadFile, "Cannot load #{file_name} file not found." unless(File.exists?(file_name))
         
+      logger.info("Perform Load Options:\n#{options.inspect}")
+      
       ext = File.extname(file_name)
-          
+      
+      # TODO - make more modular - these methods doing too much, for example move the object creation/reset
+      # out of these perform... methods to make it easier to over ride that behaviour
       if(ext.casecmp('.xls') == 0)
         perform_excel_load(file_name, options)
       elsif(ext.casecmp('.csv') == 0)
@@ -111,7 +115,9 @@ module DataShift
       end
     end
 
-    
+    def report
+      @reporter.report 
+    end
     
     # Core API
     # 
@@ -166,6 +172,7 @@ module DataShift
     def process_missing_columns_with_defaults()
       inbound_ops = @method_mapper.operator_names
       @default_values.each do |dn, dv|     
+        logger.debug "Processing default value #{dn} : #{dv}"
         assignment(dn, @load_object, dv) unless(inbound_ops.include?(dn))
       end
     end
@@ -382,20 +389,27 @@ module DataShift
       end
     end
     
-    def failure
-      @failed_objects << @load_object unless( @load_object.nil? || @load_object.new_record? || @failed_objects.include?(@load_object))
+    
+    # Loading failed. Store a failed object and if requested roll back (destroy) the current load object
+    # For use case where object saved early but subsequent required columns fail to process
+    # so the load object is invalid
+    
+    def failure( object = @load_object, rollback = false)
+      if(object)
+        @reporter.add_failed_object(object)
+      
+        @load_object.destroy if(rollback && ! @load_object.new_record?)
+        
+        new_load_object # don't forget to reset the load object 
+      end
     end
 
     def save
       return unless( @load_object )
       
-      puts "DEBUG: SAVING #{@load_object.class} : #{@load_object.inspect}" if(@verbose)
+      #puts "DEBUG: SAVING #{@load_object.class} : #{@load_object.inspect}" if(@verbose)
       begin
-        result = @load_object.save
-        
-        @loaded_objects << @load_object unless(@loaded_objects.include?(@load_object))
-
-        return result
+        return @load_object.save
       rescue => e
         failure
         puts "Error saving #{@load_object.class} : #{e.inspect}"
@@ -447,7 +461,7 @@ module DataShift
     #
     def reset(object = nil)
       @load_object = object || new_load_object
-      @loaded_objects, @failed_objects = [],[]
+      @reporter.reset
       @current_value = nil
     end
 
@@ -462,11 +476,11 @@ module DataShift
     end
 
     def loaded_count
-      @loaded_objects.size
+      reporter.loaded_objects.size
     end
 
     def failed_count
-      @failed_objects.size
+      reporter.failed_objects.size
     end
 
 

data/lib/loaders/paperclip/attachment_loader.rb

@@ -98,10 +98,14 @@ module DataShift
       
       # This version creates attachments and also attaches them to instances of :attach_to_klazz
       # 
+      # Each file found in PATH will be processed - it's filename being used to scan for
+      # a matching record to attach the file to.
+      # 
       # Options
       #   :split_file_name_on   Used in scan process to progresivly split filename to find   
       #                         :attach_to_klass with matching :attach_to_find_by_field
       #
+      #   :add_prefix
       #
       def process_from_filesystem(path, options = {} )
      
@@ -135,12 +139,11 @@ module DataShift
             
           record = nil
             
-          puts "Attempting to find matching record where #{attach_to_find_by_field} ~=  #{base_name}"
+          puts "Attempting fo find Record for file name : #{base_name}"
           record = get_record_by(attach_to_klass, attach_to_find_by_field, base_name, split_on, options)
              
           if(record)
-            puts "Found record for attachment :\n#{record.inspect}"
-            logger.info "Found record for attachment : #{record.inspect}"
+            puts "Found #{record.class} where : #{attach_to_find_by_field} = #{record.send(attach_to_find_by_field)}(id : #{record.id})"
           else
             missing_records << file_name
           end
@@ -162,15 +165,15 @@ module DataShift
           FileUtils.mkdir_p('MissingAttachmentRecords') unless File.directory?('MissingAttachmentRecords')
         
           puts "WARNING : #{missing_records.size} of #{loading_files_cache.size} files could not be attached to a #{@load_object_class}"
-          puts "For your convenience files with MISSING #{attach_to_klass} have been copied to : MissingAttachmentRecords"
-          missing_records.each do |i|
-            puts "Copying #{i} to MissingAttachmentRecords folder" if(options[:verbose])
+          puts "For your convenience copying files with MISSING #{attach_to_klass} to : MissingAttachmentRecords"
+          missing_records.each do |i| 
             FileUtils.cp( i, 'MissingAttachmentRecords')  unless(options[:dummy] == 'true')
+            puts "Copyied #{i} to MissingAttachmentRecords folder" if(options[:verbose])
           end
-        else
-          puts "Created #{loading_files_cache.size} #{@load_object_class} attachments and succesfully attached to a #{@attach_to_klass}"
         end
 
+        puts "Created #{loading_files_cache.size - missing_records.size} of #{loading_files_cache.size} #{@load_object_class} attachments and succesfully attached to a #{@attach_to_klass}"
+         
         puts "Dummy Run Complete- if happy run without -d" if(options[:dummy])
    
       end

data/lib/loaders/paperclip/datashift_paperclip.rb

@@ -13,7 +13,7 @@ module DataShift
   module Paperclip
     
     include DataShift::Logging
-    include DataShift::Logging
+  
     require 'paperclip/attachment_loader'
     
     attr_accessor :attachment
@@ -34,14 +34,14 @@ module DataShift
       
       unless File.exists?(attachment_path) && File.readable?(attachment_path)
         logger.error("Cannot process Image from #{Dir.pwd}: Invalid Path #{attachment_path}")
-        raise "Cannot process Image : Invalid Path #{attachment_path}"
+        raise PathError.new("Cannot process Image : Invalid Path #{attachment_path}")
       end
      
       file = begin
         File.new(attachment_path, "rb")
       rescue => e
         puts e.inspect
-        raise "ERROR : Failed to read image #{attachment_path}"
+        raise PathError.new("ERROR : Failed to read image from #{attachment_path}")
       end
       
       file
@@ -54,11 +54,13 @@ module DataShift
     # 
     #   :attributes
     #     
-    #     Pass through hash of attributes to klass initializer
+    #     Pass through a hash of attributes to the Paperclip klass's initializer
     # 
     #   :has_attached_file_name
     #   
     #     Paperclip attachment name defined with macro 'has_attached_file :name'  
+    #     
+    #     This is usually called/defaults  :attachment
     #   
     #     e.g 
     #       When : has_attached_file :avatar 
@@ -73,19 +75,21 @@ module DataShift
        
       has_attached_file_attribute = options[:has_attached_file_name] ? options[:has_attached_file_name].to_sym : :attachment
   
-      attributes = { has_attached_file_attribute => get_file(attachment_path) }
-
-      attributes.merge!(options[:attributes]) if(options[:attributes])
+      # e.g  (:attachment => File.read)
+      paperclip_attributes = { has_attached_file_attribute => get_file(attachment_path) }
+        
+      paperclip_attributes.merge!(options[:attributes]) if(options[:attributes])
       
-      # e.g  (:viewable => some_product) =  Icon
+      begin     
+        @attachment = klass.new(paperclip_attributes, :without_protection => true) 
+      rescue => e
+        puts e.inspect
+        logger.error("Failed to create PaperClip Attachment : #{e.inspect}")
+        raise CreateAttachmentFailed.new("Failed to create PaperClip Attachment from : #{attachment_path}")
+      end
       
-      #attributes.merge!(attach_to_record_field.to_sym => record) if(record && attach_to_record_field)
-       
-      puts attributes.inspect
       begin
         
-        @attachment = klass.new(attributes, :without_protection => true) 
-      
         if(@attachment.save)
           puts "Success: Created Attachment #{@attachment.id} : #{@attachment.attachment_file_name}"
                 
@@ -103,12 +107,11 @@ module DataShift
         
         @attachment
       rescue => e
-        puts "PaperClip error - Problem creating Attachment from : #{attachment_path}"
-        puts e.inspect, e.backtrace
+        logger.error("Problem saving Paperclip Attachment: #{e.inspect}")
+        puts e.inspect
+        raise CreateAttachmentFailed.new("PaperClip error - Problem saving Attachment")
       end
-    end
-    
+    end   
   end
   
-  
 end

data/lib/loaders/reporter.rb

@@ -0,0 +1,48 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Dec 2012
+# License::   MIT
+#
+# Details::   Store and report stats
+#
+module DataShift
+
+
+  class Reporter
+
+    include DataShift::Logging
+      
+    attr_accessor :processed_object_count, :loaded_objects, :failed_objects
+
+   
+    def initialize()
+      reset
+    end
+
+    def reset()
+      @processed_object_count = 0
+      @loaded_objects, @failed_objects = [], []
+    end
+
+    def add_loaded_object(object)
+      @loaded_objects << object unless(object.nil? || @loaded_objects.include?(object))
+    end
+    
+    def add_failed_object(object)
+      @failed_objects << object unless( object.nil? || @failed_objects.include?(object))
+    end
+    
+    def report
+      loaded_objects.compact! if(loaded_objects)
+      
+      puts "Processing complete:"
+      puts "Processed total of #{processed_object_count} entries"
+      puts "#{loaded_objects.size} objects were succesfully processed."
+       
+      puts "There were NO failures." if failed_objects.empty?
+        
+      puts "WARNING : Check logs : #{failed_objects.size} rows contained errors and #{failed_objects.size} records NOT created." unless failed_objects.empty?
+    end
+    
+  end
+end