datashift 0.10.1 → 0.10.2

data/lib/datashift/populator.rb

@@ -18,6 +18,29 @@ module DataShift
       @insistent_method_list
     end
  
+    def self.insistent_assignment( record, value, operator )
+      
+      #puts "DEBUG: RECORD CLASS #{record.class}"
+      op = operator + '='
+    
+      begin
+        record.send(op, value)
+      rescue => e
+        Populator::insistent_method_list.each do |f|
+          begin
+            record.send(op, value.send( f) )
+            break
+          rescue => e
+            #puts "DEBUG: insistent_assignment: #{e.inspect}"
+            if f == Populator::insistent_method_list.last
+              puts  "I'm sorry I have failed to assign [#{value}] to #{operator}"
+              raise "I'm sorry I have failed to assign [#{value}] to #{operator}" unless value.nil?
+            end
+          end
+        end
+      end
+    end
+    
     def assignment( operator, record, value )
       #puts "DEBUG: RECORD CLASS #{record.class}"
       op = operator + '=' unless(operator.include?('='))

data/lib/datashift/querying.rb

@@ -0,0 +1,86 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2010
+# License::   MIT
+#
+#  Details::  Base class for loaders, providing a process hook which populates a model,
+#             based on a method map and supplied value from a file - i.e a single column/row's string value.
+#             Note that although a single column, the string can be formatted to contain multiple values.
+#
+#             Tightly coupled with MethodMapper classes (in lib/engine) which contains full details of
+#             a file's column and it's correlated AR associations.
+#
+module DataShift
+
+  require 'datashift/method_mapper'
+
+  module  Querying
+ 
+    def search_for_record(klazz, field, search_term, options = {})
+    
+      begin
+        
+        if(options[:case_sensitive]) 
+          return klazz.send("find_by_#{field}", search_term)
+        elsif(options[:use_like])
+          return klazz.where("#{field} like ?", "#{search_term}%").first
+        else
+          return klazz.where("lower(#{field}) = ?", search_term.downcase).first
+        end
+     
+      rescue => e
+        puts e.inspect
+        logger.error("Exception attempting to find a record for [#{search_term}] on #{klazz}.#{field}")
+        logger.error e.backtrace
+        logger.error e.inspect
+      end
+      
+      nil
+    end
+    
+    # Find a record for model klazz, looking up on field containing search_terms
+    # Responds to global Options :
+    #   :case_sensitive : Default is a case insensitive lookup.
+    #   :use_like : Attempts a lookup using ike and x% rather than equality 
+    #
+    # Returns nil if no record found
+    def get_record_by(klazz, field, search_term, split_on = ' ', split_on_prefix = nil)
+    
+      begin
+              
+        record = search_for_record(klazz, field, search_term)
+        
+        # try individual portions of search_term, front -> back i.e "A_B_C_D" => A, B, C etc
+        search_term.split(split_on).each do |str|
+          z = (split_on_prefix) ? "#{split_on_prefix}#{str}": str
+          record = search_for_record(klazz, field, z)
+          break if record
+        end unless(record)
+        
+        # this time try incrementally scanning i.e "A_B_C_D" => A, A_B, A_B_C etc
+        search_term.split(split_on).inject("") do |str, term|
+          z = (split_on_prefix) ? "#{split_on_prefix}#{str}#{split_on}#{term}": "#{str}#{split_on}#{term}"
+          record = search_for_record(klazz, field, z)
+          break if record
+          term
+        end unless(record)
+        
+        return record
+      rescue => e
+        logger.error("Exception attempting to find a record for [#{search_term}] on #{klazz}.#{field}")
+        logger.error e.backtrace
+        logger.error e.inspect
+        return nil
+      end
+    end
+    
+    def get_record_by!(klazz, field, search_terms, split_on = ' ', split_on_prefix = nil)
+      x = get_record_by(klazz, field, search_terms, split_on, split_on_prefix)
+      
+      raise RecordNotFound, "No #{klazz} record found for [#{search_terms}] on #{field}" unless(x)
+      
+      x
+    end
+  end
+
+end

data/lib/generators/csv_generator.rb

@@ -12,11 +12,8 @@ module DataShift
 
   class CsvGenerator < GeneratorBase
 
-    attr_accessor :excel, :filename
-  
     def initialize(filename)
-      @excel = nil
-      @filename = filename
+      super(filename)
     end
 
     # Create CSV file representing supplied Model

data/lib/generators/excel_generator.rb

@@ -8,20 +8,19 @@
 # TOD : Can we switch between .xls and XSSF (POI implementation of Excel 2007 OOXML (.xlsx) file format.)
 #
 #
+require 'generator_base'
+require 'excel'
+  
 module DataShift
-
-  require 'generator_base'
       
-  require 'excel'
-
   class ExcelGenerator < GeneratorBase
 
     include DataShift::Logging
       
-    attr_accessor :excel, :filename
+    attr_accessor :excel
   
     def initialize(filename)
-      @filename = filename
+      super(filename)
     end
 
     # Create an Excel file template (header row) representing supplied Model
@@ -31,8 +30,14 @@ module DataShift
     def generate(klass, options = {})
      
       prepare_excel(klass, options)
-        
-      @excel.set_headers(MethodDictionary.assignments[klass])
+
+      prep_remove_list(options)  
+            
+      @headers = MethodDictionary.assignments[klass]
+      
+      @headers.delete_if{|h| @remove_list.include?( h.to_sym ) }
+      
+      @excel.set_headers( @headers )
 
       logger.info("ExcelGenerator saving generated template #{@filename}")
         
@@ -57,6 +62,9 @@ module DataShift
     #       
     # * <tt>:remove</tt> - Association NAME(s) to remove .. :title, :id, :name
     # .
+    # * <tt>:remove_rails</tt> - Remove Rails DB columns :
+    #           :id, :created_at, :created_on, :updated_at, :updated_on
+    #   
     def generate_with_associations(klass, options = {})
 
       prepare_excel(klass, options)
@@ -65,9 +73,9 @@ module DataShift
            
       work_list = MethodDetail::supported_types_enum.to_a - [ *options[:exclude] ]
         
-      remove_list = [ *options[:remove] ].compact.collect{|x| x.to_s.downcase.to_sym }
-         
-      headers = []
+      prep_remove_list(options)    
+      
+      @headers = []
       
       details_mgr = MethodDictionary.method_details_mgrs[klass]
                   
@@ -94,6 +102,15 @@ module DataShift
       
     private
       
+    # Take options and create a list of symbols to remove from headers
+    # 
+    def prep_remove_list( options )
+      @remove_list = [ *options[:remove] ].compact.collect{|x| x.to_s.downcase.to_sym }
+         
+      @remove_list += GeneratorBase::rails_columns if(options[:remove_rails])
+    end
+    
+    
     def prepare_excel(klass, options = {})
       @filename = options[:filename] if  options[:filename]
         

data/lib/generators/generator_base.rb

@@ -9,6 +9,18 @@ module DataShift
 
   class GeneratorBase
 
+    attr_accessor :filename, :headers, :remove_list
+  
+    def initialize(filename)
+      @filename = filename
+      @headers = []
+      @remove_list =[]
+    end
+    
+    
+    def self.rails_columns
+      @rails_standard_columns ||= [:id, :created_at, :created_on, :updated_at, :updated_on]
+    end
   end
 
 end

data/lib/loaders/csv_loader.rb

@@ -16,8 +16,14 @@ module DataShift
     
     include DataShift::Logging
     
-    # Options :
-    #   strict : Raise exception if any column cannot be mapped
+    #   Assumes header_row is first row i.e row 0
+    #   
+    #  Options passed through  to :  populate_method_mapper_from_headers
+    #  
+    #   [:mandatory]       : Array of mandatory column names
+    #   [:force_inclusion] : Array of inbound column names to force into mapping
+    #   [:include_all]     : Include all headers in processing - takes precedence of :force_inclusion
+    #   [:strict]          : Raise exception when no mapping found for a column heading (non mandatory)
     
     def perform_csv_load(file_name, options = {})
        
@@ -30,7 +36,7 @@ module DataShift
 
       # Create a method_mapper which maps list of headers into suitable calls on the Active Record class
       # For example if model has an attribute 'price' will map columns called Price, price, PRICE etc to this attribute
-      map_headers_to_operators( @parsed_file.shift, options)
+      populate_method_mapper_from_headers( @parsed_file.shift, options)
 
       puts "\n\n\nLoading from CSV file: #{file_name}"
       puts "Processing #{@parsed_file.size} rows"

data/lib/loaders/excel_loader.rb

@@ -24,8 +24,12 @@ module DataShift
     #  Options:
     #   [:sheet_number]    : Default is 0. The index of the Excel Worksheet to use.
     #   [:header_row]      : Default is 0. Use alternative row as header definition.
+    #   
+    #  Options passed through  to :  populate_method_mapper_from_headers
+    #  
     #   [:mandatory]       : Array of mandatory column names
     #   [:force_inclusion] : Array of inbound column names to force into mapping
+    #   [:include_all]     : Include all headers in processing - takes precedence of :force_inclusion
     #   [:strict]          : Raise exception when no mapping found for a column heading (non mandatory)
 
     def perform_excel_load( file_name, options = {} )
@@ -66,7 +70,9 @@ module DataShift
       
       # Create a method_mapper which maps list of headers into suitable calls on the Active Record class
       # For example if model has an attribute 'price' will map columns called Price, price, PRICE etc to this attribute
-      map_headers_to_operators( @headers, options )
+      populate_method_mapper_from_headers( @headers, options )
+      
+      @method_mapper
 
       logger.info "Excel Loader processing #{@sheet.num_rows} rows"
       
@@ -97,11 +103,13 @@ module DataShift
           # as part of this we also attempt to save early, for example before assigning to
           # has_and_belongs_to associations which require the load_object has an id for the join table
          
-          # Iterate over the columns method_mapper found in Excel,
-          # pulling data out of associated column
-          @method_mapper.method_details.each_with_index do |method_detail, col|
+          # Iterate over method_details, working on data out of associated Excel column
+          @method_mapper.method_details.each do |method_detail|
+            
+            
+            next unless method_detail # TODO populate unmapped with a real MethodDetail that is 'null' and create is_nil
 
-            value = row[col]
+            value = row[method_detail.column_index]
 
             contains_data = true unless(value.nil? || value.to_s.empty?)
               
@@ -118,7 +126,7 @@ module DataShift
           unless(save)
             failure
             logger.error "Failed to save row [#{row}]"
-            logger.error load_object.errors.inspect
+            logger.error load_object.errors.inspect if(load_object)
           else
             logger.info "Row #{row} succesfully SAVED : ID #{load_object.id}"
           end

data/lib/loaders/loader_base.rb

@@ -13,11 +13,13 @@
 module DataShift
 
   require 'datashift/method_mapper'
+  require 'datashift/querying'
 
   class LoaderBase
 
     include DataShift::Logging
     include DataShift::Populator
+    include DataShift::Querying
       
     attr_reader :headers
 
@@ -32,61 +34,7 @@ module DataShift
 
     def options() return @config; end
     
-    # Support multiple associations being added to a base object to be specified in a single column.
-    # 
-    # Entry represents the association to find via supplied name, value to use in the lookup.
-    # Can contain multiple lookup name/value pairs, separated by multi_assoc_delim ( | )
-    # 
-    # Default syntax :
-    #
-    #   Name1:value1, value2|Name2:value1, value2, value3|Name3:value1, value2
-    #
-    # E.G.
-    #   Association Properties, has a column named Size, and another called Colour,
-    #   and this combination could be used to lookup multiple associations to add to the main model Jumper
-    #
-    #       Size:small            # => generates find_by_size( 'small' )
-    #       Size:large            # => generates find_by_size( 'large' )
-    #       Colour:red,green,blue # => generates find_all_by_colour( ['red','green','blue'] )
-    #
-    #       Size:large|Size:medium|Size:large
-    #         => Find 3 different associations, perform lookup via column called Size
-    #         => Jumper.properties << [ small, medium, large ]
-    #
-    def self.name_value_delim
-      @name_value_delim ||= ':'
-      @name_value_delim
-    end
-
-    def self.set_name_value_delim(x)  @name_value_delim = x; end
-    # TODO - support embedded object creation/update via hash (which hopefully we should be able to just forward to AR)
-    #
-    #      |Category|
-    #      name:new{ :date => '20110102', :owner = > 'blah'}
-    #
-    
-    
-    def self.multi_value_delim
-      @multi_value_delim ||= ','
-      @multi_value_delim
-    end
-    
-    def self.set_multi_value_delim(x) @multi_value_delim = x; end
-    
-    # TODO - support multi embedded object creation/update via hash (which hopefully we should be able to just forward to AR)
-    #
-    #      |Category|
-    #      name:new{ :a => 1, :b => 2}|name:medium{ :a => 6, :b => 34}|name:old{ :a => 12, :b => 67}
-    #
-    def self.multi_assoc_delim
-      @multi_assoc_delim ||= '|'
-      @multi_assoc_delim
-    end
-
-
-    def self.set_multi_assoc_delim(x) @multi_assoc_delim = x; end
 
-    
     # Setup loading
     # 
     # Options to drive building the method dictionary for a class, enabling headers to be mapped to operators on that class.
@@ -109,7 +57,7 @@ module DataShift
         # Create dictionary of data on all possible 'setter' methods which can be used to
         # populate or integrate an object of type @load_object_class
         DataShift::MethodDictionary.build_method_details(@load_object_class)
-      end unless(options[:load] == false)
+      end if(options[:load] || options[:reload])
       
       @method_mapper = DataShift::MethodMapper.new
       @config = options.dup    # clone can cause issues like 'can't modify frozen hash'
@@ -125,7 +73,9 @@ module DataShift
       @prefixes       = {}
       @postfixes      = {}
       
+      # TODO - move to own LoadStats or LoadReport class
       @loaded_objects = []
+      @failed_objects = []
       
       reset(object)
     end
@@ -163,30 +113,34 @@ module DataShift
 
     
     
-    # Core API - Given a list of free text column names from a file, 
-    # map all headers to a method detail containing operator details.
+    # Core API
+    # 
+    # Given a list of free text column names from a file, 
+    # map all headers to a MethodDetail instance containing details on operator, look ups etc.
     # 
-    # This is then available through @method_mapper.method_details.each
+    # These are available through @method_mapper.method_details
     # 
     # Options:
-    #  strict           : Raise an exception of any headers can't be mapped to an attribute/association
-    #  ignore           : List of column headers to ignore when building operator map
-    #  mandatory        : List of columns that must be present in headers
+    #    [:strict]          : Raise an exception of any headers can't be mapped to an attribute/association
+    #    [:ignore]          : List of column headers to ignore when building operator map
+    #    [:mandatory]       : List of columns that must be present in headers
     #  
-    #  force_inclusion  : List of columns that do not map to any operator but should be includeed in processing.
-    #                       This provides the opportunity for loaders to provide specific methods to handle these fields
-    #                       when no direct operator is available on the modle or it's associations
+    #    [:force_inclusion] : List of columns that do not map to any operator but should be includeed in processing.
+    #                     
+    #       This provides the opportunity for loaders to provide specific methods to handle these fields
+    #       when no direct operator is available on the modle or it's associations
     #
-    def map_headers_to_operators( headers, options = {} )
+    #    [:include_all]     : Include all headers in processing - takes precedence of :force_inclusion
+    #
+    def populate_method_mapper_from_headers( headers, options = {} )
       @headers = headers
       
       mandatory = options[:mandatory] || []
-            
-      
+               
       strict = (options[:strict] == true)
       
       begin 
-        @method_mapper.map_inbound_to_methods( load_object_class, @headers, options )
+        @method_mapper.map_inbound_headers_to_methods( load_object_class, @headers, options )
       rescue => e
         puts e.inspect, e.backtrace
         logger.error("Failed to map header row to set of database operators : #{e.inspect}")
@@ -194,7 +148,7 @@ module DataShift
       end
       
       unless(@method_mapper.missing_methods.empty?)
-        puts "WARNING: These headings couldn't be mapped to class #{load_object_class} : #{@method_mapper.missing_methods.inspect}"
+        puts "WARNING: These headings couldn't be mapped to class #{load_object_class} :\n#{@method_mapper.missing_methods.inspect}"
         raise MappingDefinitionError, "Missing mappings for columns : #{@method_mapper.missing_methods.join(",")}" if(strict)
       end
 
@@ -202,6 +156,8 @@ module DataShift
         @method_mapper.missing_mandatory(mandatory).each { |e| puts "ERROR: Mandatory column missing - expected column '#{e}'" }
         raise MissingMandatoryError, "Mandatory columns missing  - please fix and retry."
       end unless(mandatory.empty?)
+      
+      @method_mapper
     end
 
 
@@ -219,59 +175,20 @@ module DataShift
     # 
     # If suitable association found, process row data and then assign to current load_object
     def find_and_process(column_name, data)
+      
+      puts "WARNING: MethodDictionary empty for class #{load_object_class}" unless(MethodDictionary.for?(load_object_class))
+        
       method_detail = MethodDictionary.find_method_detail( load_object_class, column_name )
 
       if(method_detail)
         prepare_data(method_detail, data)
         process()
       else
-        @load_object.errors.add_base( "No matching method found for column #{column_name}")
+        @load_object.errors.add(:base, "No matching method found for column #{column_name}")
       end
     end
     
     
-    # Find a record for model klazz, looking up on field containing search_terms
-    # Responds to global Options :
-    #   :case_sensitive : Default is a case insensitive lookup.
-    #   :use_like : Attempts a lookup using ike and x% ratehr than equality 
-    
-    def get_record_by(klazz, field, search_terms, split_on = ' ', split_on_prefix = nil)
-    
-      begin
-        record = if(@config[:case_sensitive]) 
-          klazz.send("find_by_#{field}", search_terms)
-        elsif(@config[:use_like])
-          klazz.where("#{field} like ?", "#{search_terms}%").first
-        else
-          klazz.where("lower(#{field}) = ?", search_terms.downcase).first
-        end
-    
-        # try the separate individual portions of the search_terms, front -> back
-        search_terms.split(split_on).each do |x|
-          z = "#{split_on_prefix}#{x}" if(split_on_prefix)
-         
-          record = get_record_by(klazz, field, z, split_on, split_on_prefix)
-          break if record
-        end unless(record)
-            
-        # this time try sequentially and incrementally scanning
-        search_terms.split(split_on).inject("") do |str, term|
-          z = (split_on_prefix) ? "#{split_on_prefix}#{str}#{term}": "#{str}#{term}"
-          record = get_record_by(klazz, field, z, split_on, split_on_prefix)
-          break if record
-          term
-        end unless(record)      
-                
-        return record 
-        
-      rescue => e
-        logger.error("Exception attempting to find a record for [#{search_terms}] on #{klazz}.#{field}")
-        logger.error e.backtrace
-        logger.error e.inspect
-        return nil
-      end
-    end
-      
     # Default values and over rides can be provided in YAML config file.
     # 
     # Any Config under key 'LoaderBase' is merged over existing options - taking precedence.
@@ -377,7 +294,7 @@ module DataShift
     #
     def get_find_operator_and_rest(inbound_data)
         
-      operator, rest = inbound_data.split(LoaderBase::name_value_delim) 
+      operator, rest = inbound_data.split(Delimiters::name_value_delim) 
      
       #puts "DEBUG inbound_data: #{inbound_data} => #{operator} , #{rest}"
        
@@ -415,23 +332,17 @@ module DataShift
 
           # A single column can contain multiple associations delimited by special char
           # Size:large|Colour:red,green,blue => ['Size:large', 'Colour:red,green,blue']
-          columns = @current_value.to_s.split( LoaderBase::multi_assoc_delim)
+          columns = @current_value.to_s.split( Delimiters::multi_assoc_delim)
 
           # Size:large|Colour:red,green,blue   => generates find_by_size( 'large' ) and find_all_by_colour( ['red','green','blue'] )
 
           columns.each do |col_str|
             
             find_operator, col_values = get_find_operator_and_rest( col_str )
-            
-            #if(@current_method_detail.find_by_operator)
-            #   find_operator, col_values = @current_method_detail.find_by_operator, col_str
-            #  else
-            #    find_operator, col_values = col_str.split(LoaderBase::name_value_delim) 
-            #  end
-          
+                      
             raise "Cannot perform DB find by #{find_operator}. Expected format key:value" unless(find_operator && col_values)
              
-            find_by_values = col_values.split(LoaderBase::multi_value_delim)
+            find_by_values = col_values.split(Delimiters::multi_value_delim)
             
             find_by_values << @current_method_detail.find_by_value if(@current_method_detail.find_by_value)
                      
@@ -471,11 +382,13 @@ module DataShift
     end
     
     def failure
-      @failed_objects << @load_object unless( !load_object.new_record? || @failed_objects.include?(@load_object))
+      @failed_objects << @load_object unless( @load_object.nil? || @load_object.new_record? || @failed_objects.include?(@load_object))
     end
 
     def save
-      puts "DEBUG: SAVING #{load_object.class} : #{load_object.inspect}" if(@verbose)
+      return unless( @load_object )
+      
+      puts "DEBUG: SAVING #{@load_object.class} : #{@load_object.inspect}" if(@verbose)
       begin
         result = @load_object.save
         
@@ -582,7 +495,7 @@ module DataShift
     # Supported Syntax :
     #  assoc_find_name:value | assoc2_find_name:value | etc
     def get_each_assoc
-      current_value.to_s.split( LoaderBase::multi_assoc_delim )
+      current_value.to_s.split( Delimiters::multi_assoc_delim )
     end
       
     private