datashift 0.5.0 → 0.6.0

data/Rakefile

@@ -38,6 +38,9 @@ Jeweler::Tasks.new do |gem|
   gem.email = "rubygems@autotelik.co.uk"
   gem.authors = ["Thomas Statter"]
   # dependencies defined in Gemfile
+  gem.files.exclude ['sandbox']
+  
+  gem.add_dependency ['spreadsheet']
 end
 Jeweler::RubygemsDotOrgTasks.new
 

data/VERSION

@@ -1 +1 @@
-0.5.0
+0.6.0

data/datashift.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "datashift"
-  s.version = "0.5.0"
+  s.version = "0.6.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Thomas Statter"]
-  s.date = "2012-05-08"
+  s.date = "2012-05-18"
   s.description = "A suite of tools to move data between ActiveRecord models,databases,applications like Excel/Open Office, files and projects including Spree"
   s.email = "rubygems@autotelik.co.uk"
   s.extra_rdoc_files = [
@@ -24,7 +24,6 @@ Gem::Specification.new do |s|
     "README.rdoc",
     "Rakefile",
     "VERSION",
-    "datashift-0.5.0.gem",
     "datashift.gemspec",
     "lib/applications/jruby/jexcel_file.rb",
     "lib/applications/jruby/word.rb",
@@ -122,7 +121,7 @@ Gem::Specification.new do |s|
     "spec/fixtures/negative/SpreeProdMissManyMandatory.xls",
     "spec/fixtures/simple_export_spec.xls",
     "spec/fixtures/simple_template_spec.xls",
-    "spec/fixtures/spree/SpreeProductImages.xls",
+    "spec/fixtures/spree/SpreeImages.xls",
     "spec/fixtures/spree/SpreeProducts.csv",
     "spec/fixtures/spree/SpreeProducts.xls",
     "spec/fixtures/spree/SpreeProductsDefaults.yml",
@@ -167,9 +166,12 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<spreadsheet>, [">= 0"])
     else
+      s.add_dependency(%q<spreadsheet>, [">= 0"])
     end
   else
+    s.add_dependency(%q<spreadsheet>, [">= 0"])
   end
 end
 

data/lib/datashift/method_detail.rb

@@ -72,8 +72,8 @@ module DataShift
       nil
     end
 
-    def operator?(name)
-      operator == name
+    def operator?(name, case_sensitive = false)
+      case_sensitive ? operator == name : operator.downcase == name.downcase
     end
 
     # Return the operator's expected class name, if can be derived, else nil

data/lib/datashift/method_dictionary.rb

@@ -59,6 +59,7 @@ module DataShift
         if(options[:instance_methods] == true)
           setters = klass.instance_methods.grep(/\w+=/).collect {|x| x.to_s }
 
+          # TODO - Since 3.2 this seems to return lots more stuff including validations which might not be appropriate
           if(klass.respond_to? :defined_activerecord_methods)
             setters = setters - klass.defined_activerecord_methods.to_a
           end
@@ -81,6 +82,13 @@ module DataShift
       end
     end
     
+
+    def self.add( klass, operator, type = :assignment)
+      method_details_mgr = get_method_details_mgr( klass )
+      md = MethodDetail.new(operator, klass, operator, type)
+      method_details_mgr <<  md
+      return md
+    end
     
     # Build a thorough and usable picture of the operators by building dictionary of our MethodDetail
     # objects which can be used to import/export data to objects of type 'klass'
@@ -115,7 +123,7 @@ module DataShift
     #
     def self.find_method_detail( klass, external_name )
 
-      md_mgr = method_details_mgrs[klass] || MethodDetailsManager.new( klass )
+      method_details_mgr = get_method_details_mgr( klass )
          
       # md_mgr.all_available_operators.each { |l| puts "DEBUG: Mapped Method : #{l.inspect}" }
         
@@ -136,7 +144,7 @@ module DataShift
         # Try each association type, returning first that contains matching operator with name n
       
         MethodDetail::supported_types_enum.each do |t|
-          method_detail = md_mgr.find(n, t)
+          method_detail = method_details_mgr.find(n, t)
           return method_detail if(method_detail)
         end
         
@@ -158,6 +166,11 @@ module DataShift
       "#{klass.name}:#{column}"
     end
     
+    def self.get_method_details_mgr( klass )
+      method_details_mgrs[klass] || MethodDetailsManager.new( klass )
+    end
+    
+        
     def self.method_details_mgrs
       @method_details_mgrs ||= {}
       @method_details_mgrs

data/lib/datashift/method_mapper.rb

@@ -50,7 +50,10 @@ module DataShift
     # specified may not be exactly as required e.g handles capitalisation, white space, _ etc
     # Returns: Array of matching method_details
     #
-    def map_inbound_to_methods( klass, columns )
+    def map_inbound_to_methods( klass, columns, options = {} )
+      
+      forced = [*options[:force_inclusion]].compact
+      forced.collect! { |f| f.downcase }
       
       @method_details, @missing_methods = [], []
     
@@ -60,16 +63,26 @@ module DataShift
           next
         end
         
-        x, lookup = name.split(MethodMapper::column_delim) 
+        operator, lookup = name.split(MethodMapper::column_delim) 
         #puts "DEBUG: Find Method Detail for #{x}"
-        md = MethodDictionary::find_method_detail( klass, x )
+        md = MethodDictionary::find_method_detail( klass, operator )
         
         # TODO be nice if we could cheeck that the assoc on klass responds to the specified
         # lookup key now (nice n early)
         # active_record_helper = "find_by_#{lookup}"
+        if(md.nil? && forced.include?(operator.downcase))
+          md = MethodDictionary::add(klass, operator)
+        end
+        
+        if(md)
+          
+          md.find_by_operator = lookup if(lookup) # TODO and klass.x.respond_to?(active_record_helper))
+           
+          @method_details << md
+        else
+          @missing_methods << operator
+        end
         
-        md.find_by_operator = lookup if(lookup) # TODO and klass.x.respond_to?(active_record_helper))
-        md ? @method_details << md : @missing_methods << x
       end
       #@method_details.compact!  .. currently we may need to map via the index on @method_details so don't remove nils for now
       @method_details

data/lib/exporters/excel_exporter.rb

@@ -104,10 +104,9 @@ module DataShift
     end # ExcelGenerator
 
   else
-    class  ExcelExporter < ExcelBase
+    class  ExcelExporter < ExporterBase
       
       def initialize(filename)
-        @filename = filename
         raise DataShift::BadRuby, "Apologies but DataShift Excel facilities currently need JRuby. Please switch to, or install JRuby"
       end
     end

data/lib/helpers/spree_helper.rb

@@ -80,9 +80,7 @@ module DataShift
     # => Will chdir into the sandbox to load environment as need to mimic being at root of a rails project
     #    chdir back after environment loaded
     
-    #  gem('rails', '3.2.3')
-    
-    def self.boot( database_env)#, rails_version = '3.1.3' )
+    def self.boot( database_env)
      
       if( ! is_namespace_version )
         db_connect( database_env )
@@ -90,11 +88,8 @@ module DataShift
         boot_pre_1
         @dslog.info "Booted Spree using pre 1.0.0 version"
       else
-        
-
-        #em('rails', rails_version)
-        
-        db_connect( database_env )#, rails_version )  
+           
+        db_connect( database_env )
         
         @dslog.info "Booting Spree using version #{SpreeHelper::version}"
        
@@ -102,13 +97,24 @@ module DataShift
         
         store_path = Dir.pwd
         
-        Dir.chdir( File.expand_path('../../../sandbox', __FILE__) )
-
-        rails_root = File.expand_path('../../../sandbox', __FILE__)
+        spree_sanbox_app = File.expand_path('../../../sandbox', __FILE__)
+        
+        unless(File.exists?(spree_sanbox_app))
+          Dir.chdir( File.expand_path( "#{spree_sanbox_app}/..") )
+          system('rails new sandbox')
+        end
+  
+        rails_root = spree_sanbox_app
         
         $:.unshift rails_root
         
-        require 'config/environment.rb'
+        begin
+          require 'config/environment.rb'
+        rescue => e
+          #somethign in deface seems to blow up suddenly on 1.1
+          # puts e.backtrace
+          puts "Warning - Potential issue initializing Spree sanbox #{e.inspect}"
+        end
         
         Dir.chdir( store_path )
         

data/lib/loaders/csv_loader.rb

@@ -28,16 +28,9 @@ module DataShift
 
       @parsed_file = CSV.read(file_name)
 
-      @mandatory = options[:mandatory] || []
-
       # Create a method_mapper which maps list of headers into suitable calls on the Active Record class
       # For example if model has an attribute 'price' will map columns called Price, price, PRICE etc to this attribute
-      map_headers_to_operators( @parsed_file.shift, options[:strict] , @mandatory )
-        
-      unless(@method_mapper.missing_methods.empty?)
-        logger.error("Following column headings could not be mapped :\n#{@method_mapper.missing_methods.inspect}")
-        raise MappingDefinitionError, "ERROR: Missing mappings for #{@method_mapper.missing_methods.size} column headings"
-      end
+      map_headers_to_operators( @parsed_file.shift, options)
 
       #if(options[:verbose])
       puts "\n\n\nLoading from CSV file: #{file_name}"

data/lib/loaders/excel_loader.rb

@@ -25,14 +25,14 @@ module DataShift
     module ExcelLoading
 
       #  Options:
-      #   [:header_row]   : Default is 0. Use alternative row as header definition.
-      #   [:mandatory]    : Array of mandatory column names
-      #   [:strict]       : Raise exception when no mapping found for a column heading (non mandatory)
-      #   [:sheet_number]
+      #   [:sheet_number]    : Default is 0. The index of the Excel Worksheet to use.
+      #   [:header_row]      : Default is 0. Use alternative row as header definition.
+      #   [:mandatory]       : Array of mandatory column names
+      #   [:force_inclusion] : Array of inbound column names to force into mapping
+      #   [:strict]          : Raise exception when no mapping found for a column heading (non mandatory)
 
-      def perform_excel_load( file_name, options = {} )
 
-        @mandatory = options[:mandatory] || []
+      def perform_excel_load( file_name, options = {} )
 
         @excel = JExcelFile.new
 
@@ -64,7 +64,7 @@ module DataShift
 
         # Create a method_mapper which maps list of headers into suitable calls on the Active Record class
         # For example if model has an attribute 'price' will map columns called Price, price, PRICE etc to this attribute
-        map_headers_to_operators( @headers, options[:strict] , @mandatory )
+        map_headers_to_operators( @headers, options )
 
         logger.info "Excel Loader prcoessing #{@excel.num_rows} rows"
         load_object_class.transaction do

data/lib/loaders/loader_base.rb

@@ -120,8 +120,15 @@ module DataShift
     #   CSV files
     # 
     # OPTIONS :
-    #   strict : Raise exception if any column cannot be mapped
-       
+    #  
+    #  strict           : Raise an exception of any headers can't be mapped to an attribute/association
+    #  ignore           : List of column headers to ignore when building operator map
+    #  mandatory        : List of columns that must be present in headers
+    #  
+    #  force_inclusion  : List of columns that do not map to any operator but should be includeed in processing.
+    #                       This provides the opportunity for loaders to provide specific methods to handle these fields
+    #                       when no direct operator is available on the modle or it's associations
+    #
     def perform_load( file_name, options = {} )
 
       raise DataShift::BadFile, "Cannot load #{file_name} file not found." unless(File.exists?(file_name))
@@ -139,21 +146,33 @@ module DataShift
     end
 
     
+    
     # Core API - Given a list of free text column names from a file, 
     # map all headers to a method detail containing operator details.
     # 
     # This is then available through @method_mapper.method_details.each
     # 
     # Options:
-    #  strict : Raise an exception of any headers can't be mapped to an attribute/association
+    #  strict           : Raise an exception of any headers can't be mapped to an attribute/association
+    #  ignore           : List of column headers to ignore when building operator map
+    #  mandatory        : List of columns that must be present in headers
+    #  
+    #  force_inclusion  : List of columns that do not map to any operator but should be includeed in processing.
+    #                       This provides the opportunity for loaders to provide specific methods to handle these fields
+    #                       when no direct operator is available on the modle or it's associations
     #
-    def map_headers_to_operators( headers, strict, mandatory = [])
+    def map_headers_to_operators( headers, options = {} )
       @headers = headers
       
+      mandatory = options[:mandatory] || []
+            
+      
+      strict = (options[:strict] == true)
+      
       begin 
-        method_details = @method_mapper.map_inbound_to_methods( load_object_class, @headers )
+        @method_mapper.map_inbound_to_methods( load_object_class, @headers, options )
       rescue => e
-        puts e.inspect
+        puts e.inspect, e.backtrace
         logger.error("Failed to map header row to set of database operators : #{e.inspect}")
         raise MappingDefinitionError, "Failed to map header row to set of database operators"
       end
@@ -443,6 +462,15 @@ module DataShift
       end
     end
 
+    protected
+    
+    # Take current column data and split into each association
+    # Supported Syntax :
+    #  assoc_find_name:value | assoc2_find_name:value | etc
+    def get_each_assoc
+      current_value.to_s.split( LoaderBase::multi_assoc_delim )
+    end
+      
     private
 
     def save_if_new

data/lib/loaders/spreadsheet_loader.rb

@@ -12,126 +12,134 @@
 #             i.e pulls data from each column and sends to object.
 #
 require 'datashift/exceptions'
-
+    
 module DataShift
+  
+  unless(Guards::jruby?)
+  
+    require 'loaders/loader_base'
 
-  require 'loaders/loader_base'
+    module SpreadsheetLoading
 
-  module SpreadsheetLoading
+      gem 'spreadsheet'
+      require 'spreadsheet'
 
-    #  Options:
-    #   [:header_row]   : Default is 0. Use alternative row as header definition.
-    #   [:mandatory]    : Array of mandatory column names
-    #   [:strict]       : Raise exception when no mapping found for a column heading (non mandatory)
-    #   [:sheet_number]
 
-    def perform_spreadsheet_load( file_name, options = {} )
+      # Spreadsheet.client_encoding = 'UTF-8'F
+    
+      #  Options:
+      #   [:header_row]   : Default is 0. Use alternative row as header definition.
+      #   [:mandatory]    : Array of mandatory column names
+      #   [:strict]       : Raise exception when no mapping found for a column heading (non mandatory)
+      #   [:sheet_number]
 
-      @mandatory = options[:mandatory] || []
+      def perform_spreadsheet_load( file_name, options = {} )
 
-      @excel = JExcelFile.new
+        @mandatory = options[:mandatory] || []
 
-      @excel.open(file_name)
+        @excel = Spreadsheet.open file_name
         
-      #if(options[:verbose])
-      puts "\n\n\nLoading from Excel file: #{file_name}"
+        #if(options[:verbose])
+        puts "\n\n\nLoading from Excel file: #{file_name}"
 
-      sheet_number = options[:sheet_number] || 0
+        sheet_number = options[:sheet_number] || 0
 
-      @sheet = @excel.sheet( sheet_number )
+        @sheet = @excel.sheet( sheet_number )
 
-      header_row_index =  options[:header_row] || 0
-      @header_row = @sheet.getRow(header_row_index)
+        header_row_index =  options[:header_row] || 0
+        @header_row = @sheet.getRow(header_row_index)
 
-      raise MissingHeadersError, "No headers found - Check Sheet #{@sheet} is complete and Row #{header_row_index} contains headers" unless(@header_row)
+        raise MissingHeadersError, "No headers found - Check Sheet #{@sheet} is complete and Row #{header_row_index} contains headers" unless(@header_row)
 
-      @headers = []
+        @headers = []
         
-      (0..JExcelFile::MAX_COLUMNS).each do |i|
-        cell = @header_row.getCell(i)
-        break unless cell
-        header = "#{@excel.cell_value(cell).to_s}".strip
-        break if header.empty?
-        @headers << header
-      end
+        (0..JExcelFile::MAX_COLUMNS).each do |i|
+          cell = @header_row.getCell(i)
+          break unless cell
+          header = "#{@excel.cell_value(cell).to_s}".strip
+          break if header.empty?
+          @headers << header
+        end
 
-      raise MissingHeadersError, "No headers found - Check Sheet #{@sheet} is complete and Row #{header_row_index} contains headers" if(@headers.empty?)
+        raise MissingHeadersError, "No headers found - Check Sheet #{@sheet} is complete and Row #{header_row_index} contains headers" if(@headers.empty?)
 
-      # Create a method_mapper which maps list of headers into suitable calls on the Active Record class
-      map_headers_to_operators( @headers, options[:strict] , @mandatory )
+        # Create a method_mapper which maps list of headers into suitable calls on the Active Record class
+        map_headers_to_operators( @headers, options)
 
-      load_object_class.transaction do
-        @loaded_objects =  []
+        load_object_class.transaction do
+          @loaded_objects =  []
 
-        (1..@excel.num_rows).collect do |row|
+          (1..@excel.num_rows).collect do |row|
 
-          # Excel num_rows seems to return all 'visible' rows, which appears to be greater than the actual data rows
-          # (TODO - write spec to process .xls with a huge number of rows)
-          #
-          # This is rubbish but currently manually detect when actual data ends, this isn't very smart but
-          # got no better idea than ending once we hit the first completely empty row
-          break if @excel.sheet.getRow(row).nil?
+            # Excel num_rows seems to return all 'visible' rows, which appears to be greater than the actual data rows
+            # (TODO - write spec to process .xls with a huge number of rows)
+            #
+            # This is rubbish but currently manually detect when actual data ends, this isn't very smart but
+            # got no better idea than ending once we hit the first completely empty row
+            break if @excel.sheet.getRow(row).nil?
 
-          contains_data = false
+            contains_data = false
 
-          # TODO - Smart sorting of column processing order ....
-          # Does not currently ensure mandatory columns (for valid?) processed first but model needs saving
-          # before associations can be processed so user should ensure mandatory columns are prior to associations
+            # TODO - Smart sorting of column processing order ....
+            # Does not currently ensure mandatory columns (for valid?) processed first but model needs saving
+            # before associations can be processed so user should ensure mandatory columns are prior to associations
 
-          # as part of this we also attempt to save early, for example before assigning to
-          # has_and_belongs_to associations which require the load_object has an id for the join table
+            # as part of this we also attempt to save early, for example before assigning to
+            # has_and_belongs_to associations which require the load_object has an id for the join table
 
-          # Iterate over the columns method_mapper found in Excel,
-          # pulling data out of associated column
-          @method_mapper.method_details.each_with_index do |method_detail, col|
+            # Iterate over the columns method_mapper found in Excel,
+            # pulling data out of associated column
+            @method_mapper.method_details.each_with_index do |method_detail, col|
 
-            value = value_at(row, col)
+              value = value_at(row, col)
 
-            contains_data = true unless(value.nil? || value.to_s.empty?)
+              contains_data = true unless(value.nil? || value.to_s.empty?)
 
-            #puts "DEBUG: Excel process METHOD :#{method_detail.inspect}", value.inspect
-            prepare_data(method_detail, value)
+              #puts "DEBUG: Excel process METHOD :#{method_detail.inspect}", value.inspect
+              prepare_data(method_detail, value)
               
-            process()
-          end
+              process()
+            end
                           
-          break unless(contains_data == true)
+            break unless(contains_data == true)
 
-          # TODO - requirements to handle not valid ?
-          # all or nothing or carry on and dump out the exception list at end
-          #puts "DEBUG: FINAL SAVE #{load_object.inspect}"
-          save
-          #puts "DEBUG: SAVED #{load_object.inspect}"
+            # TODO - requirements to handle not valid ?
+            # all or nothing or carry on and dump out the exception list at end
+            #puts "DEBUG: FINAL SAVE #{load_object.inspect}"
+            save
+            #puts "DEBUG: SAVED #{load_object.inspect}"
 
-          # don't forget to reset the object or we'll update rather than create
-          new_load_object
+            # don't forget to reset the object or we'll update rather than create
+            new_load_object
 
+          end
         end
+        puts "Spreadsheet loading stage complete - #{loaded_objects.size} rows added."
       end
-      puts "Spreadsheet loading stage complete - #{loaded_objects.size} rows added."
-    end
 
-    def value_at(row, column)
-      @excel.get_cell_value( @excel.sheet.getRow(row), column)
+      def value_at(row, column)
+        @excel.get_cell_value( @excel.sheet.getRow(row), column)
+      end
     end
-  end
 
 
-  class SpreadsheetLoader < LoaderBase
+    class SpreadsheetLoader < LoaderBase
 
-    include SpreadsheetLoading
+      include SpreadsheetLoading
   
-    def initialize(klass, object = nil, options = {})
-      super( klass, object, options )
-      raise "Cannot load - failed to create a #{klass}" unless @load_object
-    end
+      def initialize(klass, object = nil, options = {})
+        super( klass, object, options )
+        raise "Cannot load - failed to create a #{klass}" unless @load_object
+      end
 
-    def perform_load( file_name, options = {} )
-      perform_spreadsheet_load( file_name, options )
+      def perform_load( file_name, options = {} )
+        perform_spreadsheet_load( file_name, options )
 
-      puts "Spreadsheet loading stage complete - #{loaded_objects.size} rows added."  
-    end
+        puts "Spreadsheet loading stage complete - #{loaded_objects.size} rows added."  
+      end
 
-  end
+    end
   
+  end
+
 end