datashift 0.7.0 → 0.8.0

data/lib/datashift/method_detail.rb

@@ -35,7 +35,7 @@ module DataShift
     attr_reader :operator, :operator_type
 
     # TODO make it a list/primary keys
-    attr_accessor :find_by_operator
+    attr_accessor :find_by_operator, :find_by_value
     
     # Store the raw (client supplied) name against the active record  klass(model).
     # Operator is the associated method call on klass,
@@ -43,9 +43,10 @@ module DataShift
     # 
     # col_types can typically be derived from klass.columns - set of ActiveRecord::ConnectionAdapters::Column
 
-    def initialize(client_name, klass, operator, type, col_types = {}, find_by_operator = nil )
+    def initialize(client_name, klass, operator, type, col_types = {}, find_by_operator = nil, find_by_value = nil )
       @klass, @name = klass, client_name
       @find_by_operator = find_by_operator
+      @find_by_value = find_by_value
 
       if( MethodDetail::supported_types_enum.member?(type.to_sym) )
         @operator_type = type.to_sym

data/lib/datashift/method_dictionary.rb

@@ -17,6 +17,11 @@ module DataShift
     end
 
 
+    # Has the dictionary been populated for  klass
+    def self.for?(klass)
+      return !(has_many[klass] || belongs_to[klass] || has_one[klass] || assignments[klass]).nil?
+    end
+    
     # Create simple picture of all the operator names for assignment available on an AR model,
     # grouped by type of association (includes belongs_to and has_many which provides both << and = )
     # Options:
@@ -43,7 +48,7 @@ module DataShift
       #puts "Belongs To Associations:", belongs_to[klass].inspect
 
       # Find the has_one associations which can be populated via  Model.has_one_name = OtherArModelObject
-      if( options[:reload] || self.has_one[klass].nil? )
+      if( options[:reload] || has_one[klass].nil? )
         has_one[klass] = klass.reflect_on_all_associations(:has_one).map { |i| i.name.to_s }
       end
 
@@ -145,7 +150,7 @@ module DataShift
       
         MethodDetail::supported_types_enum.each do |t|
           method_detail = method_details_mgr.find(n, t)
-          return method_detail if(method_detail)
+          return method_detail.clone if(method_detail)
         end
         
       end

data/lib/datashift/method_mapper.rb

@@ -32,6 +32,9 @@ module DataShift
     # in the heading .. i.e Column header => 'BlogPosts:user_id' 
     # ... association has many BlogPosts selected via find_by_user_id
     # 
+    # in the heading .. i.e Column header => 'BlogPosts:user_name:John Smith' 
+    # ... association has many BlogPosts selected via find_by_user_name("John Smith")
+    #
     def self.column_delim
       @column_delim ||= ':'
       @column_delim
@@ -60,11 +63,12 @@ module DataShift
       columns.each do |name|
         if(name.nil? or name.empty?)
           logger.warn("Column list contains empty or null columns") 
+          @method_details << nil
           next
         end
         
         operator, lookup = name.split(MethodMapper::column_delim) 
-        #puts "DEBUG: Find Method Detail for #{x}"
+       
         md = MethodDictionary::find_method_detail( klass, operator )
         
         # TODO be nice if we could cheeck that the assoc on klass responds to the specified
@@ -76,8 +80,13 @@ module DataShift
         
         if(md)
           
-          md.find_by_operator = lookup if(lookup) # TODO and klass.x.respond_to?(active_record_helper))
-           
+          if(lookup)
+            find_by, find_value = lookup.split(MethodMapper::column_delim) 
+            md.find_by_value    = find_value 
+            md.find_by_operator = find_by # TODO and klass.x.respond_to?(active_record_helper))
+            #puts "DEBUG: Method Detail #{md.name};#{md.operator} : find_by_operator #{md.find_by_operator}"
+          end
+              
           @method_details << md
         else
           @missing_methods << operator

data/lib/datashift/model_mapper.rb

@@ -6,19 +6,19 @@ class ModelMapper
   # e.g "Spree::Property" returns the Spree::Property class
   # Raises exception if no such class found
   def self.const_get_from_string(str)
-    str.split('::').inject(Object) do |mod, class_name| 
+    str.to_s.split('::').inject(Object) do |mod, class_name| 
       mod.const_get(class_name) 
     end 
   end 
-  
-  
+
+
   # Similar to const_get_from_string except this version
   # returns nil if no such class found
   # Support modules e.g "Spree::Property"
   # 
   def self.class_from_string( str )
       begin
-        ModelMapper::const_get_from_string(str)  #Kernel.const_get(model)
+        ModelMapper::const_get_from_string(str.to_s)  #Kernel.const_get(model)
       rescue NameError => e
        return nil
       end

data/lib/generators/excel_generator.rb

@@ -27,7 +27,9 @@ module DataShift
       end
 
       # Create an Excel file template (header row) representing supplied Model
-    
+      # Options:
+      # * <tt>:autosize</tt> - Autosize all the columns
+      #
       def generate(klass, options = {})
      
         prepare(klass, options)
@@ -36,18 +38,25 @@ module DataShift
 
         logger.info("ExcelGenerator saving generated template #{@filename}")
         
+        @excel.autosize if(options[:autosize])
+        
         @excel.save( @filename )
       end
 
       
-      # Create an Excel file from list of ActiveRecord objects
-      # To remove type(s) of associations specify option :
-      #   :exclude => [type(s)]
+      # Create an Excel file template (header row) representing supplied Model
+      # and it's associations
+      # 
+      # Options:
+      # * <tt>:autosize</tt> - Autosize all the columns
+      #
+      # * <tt>:exclude</tt> - Associations to exclude.
+      #   You can specify a hash of {association_type => [array of association names] }
+      #   to exclude from the template.
       #   
-      # Possible values are given by MethodDetail::supported_types_enum
+      # Possible association_type values are given by MethodDetail::supported_types_enum
       #  ... [:assignment, :belongs_to, :has_one, :has_many]
       #
-      # Options
       def generate_with_associations(klass, options = {})
 
         prepare(klass, options)
@@ -69,6 +78,8 @@ module DataShift
         end
         
         @excel.set_headers( headers )
+        
+        @excel.autosize if(options[:autosize])
                 
         @excel.save( filename() )
       end

data/lib/loaders/excel_loader.rb

@@ -66,7 +66,7 @@ module DataShift
         # For example if model has an attribute 'price' will map columns called Price, price, PRICE etc to this attribute
         map_headers_to_operators( @headers, options )
 
-        logger.info "Excel Loader prcoessing #{@excel.num_rows} rows"
+        logger.info "Excel Loader processing #{@excel.num_rows} rows"
         load_object_class.transaction do
           @loaded_objects =  []
 
@@ -124,8 +124,9 @@ module DataShift
           end
         end
         puts "Excel loading stage complete - #{loaded_objects.size} rows added."
+        puts "There were NO failures." if failed_objects.empty?
         
-        puts "WARNING : #{failed_objects.size} rows contained errors and #{failed_objects.size} records NOT created." unless failed_objects.empty?
+        puts "WARNING : Check logs : #{failed_objects.size} rows contained errors and #{failed_objects.size} records NOT created." unless failed_objects.empty?
       end
 
       def value_at(row, column)

data/lib/loaders/loader_base.rb

@@ -85,21 +85,33 @@ module DataShift
     def self.set_multi_assoc_delim(x) @multi_assoc_delim = x; end
 
     
+    # Setup loading
+    # 
+    # Options to drive building the method dictionary for a class, enabling headers to be mapped to operators on that class.
+    #  
     # Options
-    #     :instance_methods => true
+    #  :load [default = true] : Load the method dictionary for object_class
+    #  
+    #  :reload           : Force load of the method dictionary for object_class even if already loaded
+    #  :instance_methods : Include setter type instance methods for assignment as well as AR columns
 
+    
     def initialize(object_class, object = nil, options = {})
       @load_object_class = object_class
 
       # Gather names of all possible 'setter' methods on AR class (instance variables and associations)
-      DataShift::MethodDictionary.find_operators( @load_object_class, :reload => true, :instance_methods => options[:instance_methods] )
-
-      # Create dictionary of data on all possible 'setter' methods which can be used to
-      # populate or integrate an object of type @load_object_class
-      DataShift::MethodDictionary.build_method_details(@load_object_class)
+      unless(MethodDictionary::for?(object_class) && options[:reload] == false)
+        #puts "Building Method Dictionary for class #{object_class}"
+        DataShift::MethodDictionary.find_operators( @load_object_class, :reload => options[:reload], :instance_methods => options[:instance_methods] )
+        
+        # Create dictionary of data on all possible 'setter' methods which can be used to
+        # populate or integrate an object of type @load_object_class
+        DataShift::MethodDictionary.build_method_details(@load_object_class)
+      end unless(options[:load] == false)
       
       @method_mapper = DataShift::MethodMapper.new
-      @options = options.clone
+      @options = options.dup    # clone can cause issues like 'can't modify frozen hash'
+
       @verbose = @options[:verbose]
       @headers = []
 
@@ -127,8 +139,8 @@ module DataShift
     #  mandatory        : List of columns that must be present in headers
     #  
     #  force_inclusion  : List of columns that do not map to any operator but should be includeed in processing.
-    #                       This provides the opportunity for loaders to provide specific methods to handle these fields
-    #                       when no direct operator is available on the modle or it's associations
+    #                     This provides the opportunity for loaders to provide specific methods to handle these fields
+    #                     when no direct operator is available on the model or it's associations
     #
     def perform_load( file_name, options = {} )
 
@@ -179,7 +191,7 @@ module DataShift
       end
       
       unless(@method_mapper.missing_methods.empty?)
-        puts "WARNING: Following column headings could not be mapped : #{@method_mapper.missing_methods.inspect}"
+        puts "WARNING: These headings couldn't be mapped to class #{load_object_class} : #{@method_mapper.missing_methods.inspect}"
         raise MappingDefinitionError, "Missing mappings for columns : #{@method_mapper.missing_methods.join(",")}" if(strict)
       end
 
@@ -214,18 +226,20 @@ module DataShift
       end
     end
     
-    def get_record_by(klazz, field, value)
-      x =  (@options[:sku_prefix]) ? "#{@options[:sku_prefix]}#{value}" : value
-
+    
+    # Find a record for model klazz, looking up on field for x
+    # Responds to global Options :
+    # :case_sensitive : Default is a case insensitive lookup.
+    # :use_like : Attempts a lookup using ike and x% ratehr than equality
+    
+    def get_record_by(klazz, field, x)
+    
       begin
         if(@options[:case_sensitive]) 
-          puts "Search case sensitive for [#{x}] on #{field}" if(@verbose)
           return klazz.send("find_by_#{field}", x)
         elsif(@options[:use_like])
-          puts "Term : #{klazz}.where(\"#{field} LIKE '#{x}%'\")" if(@verbose)
           return klazz.where("#{field} like ?", "#{x}%").first
         else
-          puts "Term : #{klazz}.where(\"lower(#{field}) = '#{x.downcase}'\")" if(@verbose)
           return klazz.where("lower(#{field}) = ?", x.downcase).first
         end
       rescue => e
@@ -238,8 +252,10 @@ module DataShift
       
     # Default values and over rides can be provided in YAML config file.
     # 
-    # Any Configuration under key 'LoaderBase' is merged into this classes
-    # existing options - taking precedence.
+    # Any Config under key 'LoaderBase' is merged over existing options - taking precedence.
+    #  
+    # Any Config under a key equal to the full name of the Loader class (e.g DataShift::SpreeHelper::ImageLoader)
+    # is merged over existing options - taking precedence.
     # 
     #  Format :
     #  
@@ -297,6 +313,10 @@ module DataShift
       if(data['LoaderBase'])
         @options.merge!(data['LoaderBase'])
       end
+       
+      if(data[self.class.name])    
+        @options.merge!(data[self.class.name])
+      end
       
       logger.info("Loader Options : #{@options.inspect}")
     end
@@ -326,18 +346,31 @@ module DataShift
       @current_value
     end
     
-    # return the find_by operator and the values to find
-    def get_find_operator_and_rest( column_data)
-    
-      find_operator, col_values = "",nil
-           
+    # Return the find_by operator and the rest of the (row,columns) data
+    #   price:0.99
+    # 
+    # Column headings can already contain the operator so possible that row only contains
+    #   0.99
+    # We leave it to caller to manage any other aspects or problems in 'rest'
+    #
+    def get_find_operator_and_rest(inbound_data)
+        
+      operator, rest = inbound_data.split(LoaderBase::name_value_delim) 
+     
+      #puts "DEBUG inbound_data: #{inbound_data} => #{operator} , #{rest}"
+       
+      # Find by operator embedded in row takes precedence over operator in column heading
       if(@current_method_detail.find_by_operator)
-        find_operator, col_values = @current_method_detail.find_by_operator, column_data
-      else
-        find_operator, col_values = column_data.split(LoaderBase::name_value_delim) 
+        # row contains 0.99 so rest is effectively operator, and operator is in method details
+        if(rest.nil?)
+          rest = operator
+          operator = @current_method_detail.find_by_operator
+        end
       end
        
-      return find_operator, col_values
+      #puts "DEBUG: get_find_operator_and_rest: #{operator} => #{rest}"
+      
+      return operator, rest
     end
     
     # Process a value string from a column.
@@ -377,6 +410,8 @@ module DataShift
             raise "Cannot perform DB find by #{find_operator}. Expected format key:value" unless(find_operator && col_values)
              
             find_by_values = col_values.split(LoaderBase::multi_value_delim)
+            
+            find_by_values << @current_method_detail.find_by_value if(@current_method_detail.find_by_value)
                      
             if(find_by_values.size > 1)
 

data/lib/loaders/paperclip/image_loader.rb

@@ -0,0 +1,75 @@
+# Copyright:: (c) Autotelik Media Ltd 2012
+# Author ::   Tom Statter
+# Date ::     June 2012
+# License::   MIT. Free, Open Source.
+#
+# => Provides facilities for bulk uploading/exporting attachments provided by PaperClip 
+# gem
+require 'loader_base'
+
+module DataShift
+
+  module ImageLoading
+ 
+    include DataShift::Logging
+    
+    def self.get_files(path, options = {})
+      glob = (options['recursive'] || options[:recursive])  ? "**/*.{jpg,png,gif}" : "*.{jpg,png,gif}"
+      
+      Dir.glob("#{path}/#{glob}")
+    end
+  
+    def get_file( attachment_path )
+      
+      unless File.exists?(attachment_path) && File.readable?(attachment_path)
+        logger.error("Cannot process Image from #{Dir.pwd}: Invalid Path #{attachment_path}")
+        raise "Cannot process Image : Invalid Path #{attachment_path}"
+      end
+     
+      file = begin
+        File.new(attachment_path, "rb")
+      rescue => e
+        puts e.inspect
+        raise "ERROR : Failed to read image #{attachment_path}"
+      end
+      
+      file
+    end
+    
+    # Note the paperclip attachment model defines the storage path via something like :
+    # => :path => ":rails_root/public/blah/blahs/:id/:style/:basename.:extension"
+    # Options 
+    #   has_attached_file_name : Paperclip attachment name defined with macro 'has_attached_file :name'  e.g has_attached_file :avatar
+    #
+    def create_image(klass, attachment_path, viewable_record = nil, options = {})
+       
+      has_attached_file = options[:has_attached_file_name] || :attachment
+      
+      alt = if(options[:alt])
+        options[:alt]
+      else
+        (viewable_record and viewable_record.respond_to? :name) ? viewable_record.name : ""
+      end
+    
+      position = (viewable_record and viewable_record.respond_to?(:images)) ? viewable_record.images.length : 0
+          
+      file = get_file(attachment_path)
+
+      begin
+        
+        image = klass.new( 
+          {has_attached_file.to_sym => file, :viewable => viewable_record, :alt => alt, :position => position},
+          :without_protection => true
+        )  
+        
+        #image.attachment.reprocess!  not sure this is required anymore
+        
+        puts image.save ? "Success: Created Image: #{image.id} : #{image.attachment_file_name}" : "ERROR : Problem saving to DB Image: #{image.inspect}"
+      rescue => e
+        puts "PaperClip error - Problem creating an Image from : #{attachment_path}"
+        puts e.inspect, e.backtrace
+      end
+    end
+  end
+      
+end

data/lib/loaders/spree/image_loader.rb

@@ -4,58 +4,24 @@
 # License::   MIT. Free, Open Source.
 #
 require 'loader_base'
+require 'paperclip/image_loader'
 
 module DataShift
 
 
-  module ImageLoading
+  module DataShift::SpreeImageLoading
  
     include DataShift::Logging
+    include DataShift::ImageLoading
      
-    def get_file( attachment_path )
-      
-      unless File.exists?(attachment_path) && File.readable?(attachment_path)
-        logger.error("Cannot process Image from #{Dir.pwd}: Invalid Path #{attachment_path}")
-        raise "Cannot process Image : Invalid Path #{attachment_path}"
-      end
-     
-      file = begin
-        File.new(attachment_path, "rb")
-      rescue => e
-        puts e.inspect
-        raise "ERROR : Failed to read image #{attachment_path}"
-      end
-      
-      file
-    end
-    
     # Note the Spree Image model sets default storage path to
     # => :path => ":rails_root/public/assets/products/:id/:style/:basename.:extension"
 
     def create_image(klass, attachment_path, viewable_record = nil, options = {})
-       
-      alt = if(options[:alt])
-        options[:alt]
-      else
-        (viewable_record and viewable_record.respond_to? :name) ? viewable_record.name : ""
-      end
-    
-      position = (viewable_record and viewable_record.respond_to?(:images)) ? viewable_record.images.length : 0
-          
-      file = get_file(attachment_path)
-      
-      if(SpreeHelper::version.to_f > 1 && viewable_record.is_a?(Spree::Product) )
-       
-        image = klass.new( :attachment => file, :alt => alt, :position => position)  
         
-        # mass assignment not allows for this field
-        image.viewable = viewable_record.master
-      else
-        image = klass.new( :attachment => file,:viewable => viewable_record, :alt => alt, :position => position)  
-      end
-      #image.attachment.reprocess!
-
-      puts image.save ? "Success: Created Image: #{image.inspect}" : "ERROR : Problem saving to DB Image: #{image.inspect}"
+      viewable =  (SpreeHelper::version.to_f > 1 && viewable_record.is_a?(Spree::Product) ) ? viewable_record.master : viewable_record
+      
+      super(klass, attachment_path, viewable, options)
     end
   end
 
@@ -64,12 +30,15 @@ module DataShift
     # TODO - extract this out of SpreeHelper to create  a general paperclip loader
     class ImageLoader < LoaderBase
 
-      include DataShift::ImageLoading
+      include DataShift::SpreeImageLoading
       include DataShift::CsvLoading
       include DataShift::ExcelLoading
       
       def initialize(image = nil, options = {})
-        super( SpreeHelper::get_spree_class('Image'), image, options )
+        
+        opts = options.merge(:load => false)  # Don't need operators and no table Spree::Image
+
+        super( SpreeHelper::get_spree_class('Image'), image, opts )
         
         if(SpreeHelper::version.to_f > 1.0 )
           @attachment_klazz  = DataShift::SpreeHelper::get_spree_class('Variant' )