datashift 0.2.2 → 0.4.0

data/lib/datashift/method_dictionary.rb

@@ -11,6 +11,8 @@ module DataShift
 
   class MethodDictionary
 
+    include DataShift::Logging
+    
     def initialize
     end
 
@@ -22,12 +24,15 @@ module DataShift
     #   :instance_methods => if true include instance method type assignment operators as well as model's pure columns
     #
     def self.find_operators(klass, options = {} )
+      
+      raise "Cannot find operators supplied klass nil #{klass}" if(klass.nil?)
 
       # Find the has_many associations which can be populated via <<
       if( options[:reload] || has_many[klass].nil? )
         has_many[klass] = klass.reflect_on_all_associations(:has_many).map { |i| i.name.to_s }
         klass.reflect_on_all_associations(:has_and_belongs_to_many).inject(has_many[klass]) { |x,i| x << i.name.to_s }
       end
+      
       # puts "DEBUG: Has Many Associations:", has_many[klass].inspect
 
       # Find the belongs_to associations which can be populated via  Model.belongs_to_name = OtherArModelObject
@@ -48,7 +53,7 @@ module DataShift
       # Note, not all reflections return method names in same style so we convert all to
       # the raw form i.e without the '='  for consistency 
       if( options[:reload] || assignments[klass].nil? )
-
+ 
         assignments[klass] = klass.column_names
            
         if(options[:instance_methods] == true)

data/lib/datashift/method_mapper.rb

@@ -22,6 +22,8 @@ module DataShift
 
   class MethodMapper
 
+    include DataShift::Logging
+    
     attr_accessor :header_row, :headers
     attr_accessor :method_details, :missing_methods
   
@@ -43,16 +45,21 @@ module DataShift
       @headers = []
     end
 
-    # Build complete picture of the methods whose names listed in method_list
-    # Handles method names as defined by a user or in file headers where names may
-    # not be exactly as required e.g handles capitalisation, white space, _ etc
+    # Build complete picture of the methods whose names listed in columns
+    # Handles method names as defined by a user, from spreadsheets or file headers where the names
+    # specified may not be exactly as required e.g handles capitalisation, white space, _ etc
     # Returns: Array of matching method_details
     #
-    def map_inbound_to_methods( klass, method_list )
+    def map_inbound_to_methods( klass, columns )
       
       @method_details, @missing_methods = [], []
     
-      method_list.each do |name|
+      columns.each do |name|
+        if(name.nil? or name.empty?)
+          logger.warn("Column list contains empty or null columns") 
+          next
+        end
+        
         x, lookup = name.split(MethodMapper::column_delim) 
         md = MethodDictionary::find_method_detail( klass, x )
         

data/lib/datashift/populator.rb

@@ -0,0 +1,46 @@
+# Copyright:: (c) Autotelik Media Ltd 2012
+# Author ::   Tom Statter
+# Date ::     March 2012
+# License::   MIT
+#
+# Details::   This modules provides individual population methods on an AR model.
+#
+#             Enables users to assign values to AR object, without knowing much about that receiving object.
+#
+require 'to_b'
+
+module DataShift
+
+  module Populator
+    
+    def self.insistent_method_list
+      @insistent_method_list ||= [:to_s, :to_i, :to_f, :to_b]
+      @insistent_method_list
+    end
+ 
+    def assignment( operator, record, value )
+      #puts "DEBUG: RECORD CLASS #{record.class}"
+      op = operator + '=' unless(operator.include?('='))
+    
+      begin
+        record.send(op, value)
+      rescue => e
+        Populator::insistent_method_list.each do |f|
+          begin
+            record.send(op, value.send( f) )
+            break
+          rescue => e
+            #puts "DEBUG: insistent_assignment: #{e.inspect}"
+            if f == Populator::insistent_method_list.last
+              puts  "I'm sorry I have failed to assign [#{value}] to #{operator}"
+              raise "I'm sorry I have failed to assign [#{value}] to #{operator}" unless value.nil?
+            end
+          end
+        end
+      end
+    end
+   
+    
+  end
+  
+end

data/lib/exporters/excel_exporter.rb

@@ -108,7 +108,7 @@ module DataShift
       
       def initialize(filename)
         @filename = filename
-        raise DataShift::BadRuby, "Apologies but Datashift Excel facilities currently need JRuby. Please switch to, or install JRuby"
+        raise DataShift::BadRuby, "Apologies but DataShift Excel facilities currently need JRuby. Please switch to, or install JRuby"
       end
     end
   end # jruby

data/lib/generators/excel_generator.rb

@@ -18,7 +18,9 @@ module DataShift
 
     class ExcelGenerator < GeneratorBase
 
-      attr_accessor :filename
+      include DataShift::Logging
+      
+      attr_accessor :excel, :filename
   
       def initialize(filename)
         @filename = filename
@@ -27,70 +29,72 @@ module DataShift
       # Create an Excel file template (header row) representing supplied Model
     
       def generate(klass, options = {})
-        MethodDictionary.find_operators( klass )
-
-        @filename = options[:filename] if  options[:filename]
-
-        excel = JExcelFile.new()
-
-        if(options[:sheet_name] )
-          excel.create_sheet( options[:sheet_name] ) 
-        else
-          excel.create_sheet( klass.name )
-        end
+     
+        prepare(klass, options)
         
-        raise "Failed to create Excel WorkSheet for #{klass.name}" unless excel.sheet
+        @excel.set_headers(MethodDictionary.assignments[klass])
 
-        excel.set_headers(MethodDictionary.assignments[klass])
-
-        excel.save( @filename )
+        logger.info("ExcelGenerator saving generated template #{@filename}")
+        
+        @excel.save( @filename )
       end
 
       
       # Create an Excel file from list of ActiveRecord objects
-      # Specify which associations to export via :with or :exclude
-      # Possible values are : [:assignment, :belongs_to, :has_one, :has_many]
+      # To remove type(s) of associations specify option :
+      #   :exclude => [type(s)]
+      #   
+      # Possible values are given by MethodDetail::supported_types_enum
+      #  ... [:assignment, :belongs_to, :has_one, :has_many]
       #
+      # Options
       def generate_with_associations(klass, options = {})
 
-        excel = JExcelFile.new()
-
-        if(options[:sheet_name] )
-          excel.create_sheet( options[:sheet_name] ) 
-        else
-          excel.create_sheet( klass.name )
-        end
-        
-        MethodDictionary.find_operators( klass )
+        prepare(klass, options)
          
         MethodDictionary.build_method_details( klass )
            
-        work_list = options[:with] || MethodDetail::supported_types_enum
-        
+        work_list = MethodDetail::supported_types_enum.to_a
+        work_list -= options[:exclude].to_a
+         
         headers = []
-        puts "work_list :  [#{work_list.inspect}]"
       
         details_mgr = MethodDictionary.method_details_mgrs[klass]
                   
         work_list.each do |op_type|
           list_for_class_and_op = details_mgr.get_list(op_type)
-       
+          
           next if(list_for_class_and_op.nil? || list_for_class_and_op.empty?)
-         #if(work_list.include?(md.operator_type))
-              #each do |mdtype|
-          #end
-          #if(MethodDictionary.respond_to?("#{mdtype}_for") )
-           # method_details = MethodDictionary.send("#{mdtype}_for", klass)
-        
-            list_for_class_and_op.each {|md| headers << "#{md.operator}" }
-          #else
-           # puts "ERROR : Unknown option in :with [#{mdtype}]"
-         
+          list_for_class_and_op.each {|md| headers << "#{md.operator}" }
         end
         
-        excel.set_headers( headers )
+        @excel.set_headers( headers )
                 
-        excel.save( filename() )
+        @excel.save( filename() )
+      end
+      
+      private
+      
+      def prepare(klass, options = {})
+        @filename = options[:filename] if  options[:filename]
+        
+        logger.info("ExcelGenerator creating template with associations for class #{klass}")
+        
+        @excel = JExcelFile.new()
+
+        if(options[:sheet_name] )
+          @excel.create_sheet( options[:sheet_name] ) 
+        else
+          @excel.create_sheet( klass.name )
+        end
+        
+        unless @excel.sheet
+          logger.error("Excel failed to create WorkSheet for #{klass.name}")
+        
+          raise "Failed to create Excel WorkSheet for #{klass.name}" 
+        end
+        
+        MethodDictionary.find_operators( klass )
       end
     end # ExcelGenerator
 
@@ -99,7 +103,7 @@ module DataShift
       
       def initialize(filename)
         @filename = filename
-        raise DataShift::BadRuby, "Apologies but Datashift Excel facilities currently need JRuby. Please switch to, or install JRuby"
+        raise DataShift::BadRuby, "Apologies but DataShift Excel facilities currently need JRuby. Please switch to, or install JRuby"
       end
     end
   end # jruby

data/lib/helpers/spree_helper.rb

@@ -25,8 +25,6 @@
 #             as the database is auto generated
 # =>          
 
-
-  
 module DataShift
     
   module SpreeHelper
@@ -72,10 +70,15 @@ module DataShift
     end
     
     
-    # Datahift isi usually included and tasks pulled in by a parent/host application.
+    # Datashift is usually included and tasks pulled in by a parent/host application.
     # So here we are hacking our way around the fact that datashift is not a Rails/Spree app/engine
     # so that we can ** run our specs ** directly in datashift library
     # i.e without ever having to install datashift in a host application
+    #
+    # NOTES:
+    # => Will chdir into the sandbox to load environment as need to mimic being at root of a rails project
+    #    chdir back after environment loaded
+    
     def self.boot( database_env )
      
       if( ! is_namespace_version )
@@ -96,10 +99,18 @@ module DataShift
        
         require 'rails/all'
         
+        store_path = Dir.pwd
+        
         Dir.chdir( File.expand_path('../../../sandbox', __FILE__) )
 
+        rails_root = File.expand_path('../../../sandbox', __FILE__)
+        
+        $:.unshift rails_root
+        
         require 'config/environment.rb'
         
+        Dir.chdir( store_path )
+        
         @dslog.info "Booted Spree using post 1.0.0 version"
       end
     end

data/lib/loaders/csv_loader.rb

@@ -14,8 +14,10 @@ module DataShift
      
   module CsvLoading
     
+    include DataShift::Logging
+    
     def perform_csv_load(file_name, options = {})
-
+       
       require "csv"
 
       # TODO - can we abstract out what a 'parsed file' is - so a common object can represent excel,csv etc
@@ -23,14 +25,12 @@ module DataShift
 
       @parsed_file = CSV.read(file_name)
 
-
-      @method_mapper = DataShift::MethodMapper.new
-
       @mandatory = options[:mandatory] || []
 
       # Create a method_mapper which maps list of headers into suitable calls on the Active Record class
+      # For example if model has an attribute 'price' will map columns called Price, price, PRICE etc to this attribute
       map_headers_to_operators( @parsed_file.shift, options[:strict] , @mandatory )
-
+        
       unless(@method_mapper.missing_methods.empty?)
         puts "WARNING: Following column headings could not be mapped : #{@method_mapper.missing_methods.inspect}"
         raise MappingDefinitionError, "ERROR: Missing mappings for #{@method_mapper.missing_methods.size} column headings"
@@ -45,6 +45,9 @@ module DataShift
         @loaded_objects =  []
 
         @parsed_file.each do |row|
+          
+          # First assign any default values for columns not included in parsed_file
+          process_missing_columns_with_defaults
 
           # TODO - Smart sorting of column processing order ....
           # Does not currently ensure mandatory columns (for valid?) processed first but model needs saving
@@ -67,7 +70,7 @@ module DataShift
           # TODO - handle when it's not valid ?
           # Process rest and dump out an exception list of Products ??
 
-          puts "SAVING ROW #{row} : #{load_object.inspect}" #if options[:verbose]
+          logger.info "Saving csv row #{row} to table object : #{load_object.inspect}" #if options[:verbose]
 
           save
 

data/lib/loaders/excel_loader.rb

@@ -80,6 +80,10 @@ module DataShift
             break if @excel.sheet.getRow(row).nil?
 
             contains_data = false
+            
+            # First assign any default values for columns not included in parsed_file
+            process_missing_columns_with_defaults
+
 
             # TODO - Smart sorting of column processing order ....
             # Does not currently ensure mandatory columns (for valid?) processed first but model needs saving
@@ -87,7 +91,7 @@ module DataShift
 
             # as part of this we also attempt to save early, for example before assigning to
             # has_and_belongs_to associations which require the load_object has an id for the join table
-
+         
             # Iterate over the columns method_mapper found in Excel,
             # pulling data out of associated column
             @method_mapper.method_details.each_with_index do |method_detail, col|

data/lib/loaders/loader_base.rb

@@ -16,8 +16,8 @@ module DataShift
 
   class LoaderBase
 
-    
     include DataShift::Logging
+    include DataShift::Populator
       
     attr_reader :headers
 
@@ -131,8 +131,13 @@ module DataShift
     def map_headers_to_operators( headers, strict, mandatory = [])
       @headers = headers
       
-      method_details = @method_mapper.map_inbound_to_methods( load_object_class, @headers )
-
+      begin 
+        method_details = @method_mapper.map_inbound_to_methods( load_object_class, @headers )
+      rescue => e
+        logger.error("Failed to map header row to set of database operators : #{e.inspect}")
+        raise MappingDefinitionError, "Failed to map header row to set of database operators"
+      end
+      
       unless(@method_mapper.missing_methods.empty?)
         puts "WARNING: Following column headings could not be mapped : #{@method_mapper.missing_methods.inspect}"
         raise MappingDefinitionError, "Missing mappings for columns : #{@method_mapper.missing_methods.join(",")}" if(strict)
@@ -145,6 +150,15 @@ module DataShift
     end
 
 
+    # Process any defaults user has specified, for those columns that are not included in
+    # the incoming import format
+    def process_missing_columns_with_defaults()
+      inbound_ops = @method_mapper.operator_names
+      @default_values.each do |dn, dv|     
+        assignment(dn, @load_object, dv) unless(inbound_ops.include?(dn))
+      end
+    end
+    
     # Core API - Given a single free text column name from a file, search method mapper for
     # associated operator on base object class.
     # 
@@ -176,25 +190,25 @@ module DataShift
       #   IDEAS .....
       #
       #unless(@default_data_objects[load_object_class])
-    #
-     #   @default_data_objects[load_object_class] = load_object_class.new
+      #
+      #   @default_data_objects[load_object_class] = load_object_class.new
       
       #  default_data_object = @default_data_objects[load_object_class]
       
       
-       # default_data_object.instance_eval do
-        #  def datashift_defaults=(hash)
-         #   @datashift_defaults = hash
-        #  end
-        #  def datashift_defaults
-        #    @datashift_defaults
-        #  end
-        #end unless load_object_class.respond_to?(:datashift_defaults)
+      # default_data_object.instance_eval do
+      #  def datashift_defaults=(hash)
+      #   @datashift_defaults = hash
+      #  end
+      #  def datashift_defaults
+      #    @datashift_defaults
+      #  end
+      #end unless load_object_class.respond_to?(:datashift_defaults)
       #end
       
       #puts load_object_class.new.to_yaml
       
-      puts data.inspect
+      logger.info("Read Datashift loading config: #{data.inspect}")
       
       if(data[load_object_class.name])