datashift 0.9.0 → 0.10.0

data/lib/guards.rb

@@ -0,0 +1,57 @@
+# Copyright:: (c) Autotelik Media Ltd 2010 - 2012 Tom Statter
+# Author ::   Tom Statter
+# Date ::     Aug 2010
+# License::   Free, Open Source.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+
+# Details::   Active Record Loader
+# 
+# To pull DataShift commands into your main application :
+#
+#     require 'datashift'
+#
+#     DataShift::load_commands
+#
+require 'rbconfig'
+  
+module DataShift
+
+  module Guards
+
+    def self.jruby?
+      return RUBY_PLATFORM == "java"
+    end
+    def self.mac?
+      RbConfig::CONFIG['target_os'] =~ /darwin/i
+    end
+
+    def self.linux?
+      RbConfig::CONFIG['target_os'] =~ /linux/i
+    end
+
+    def self.windows?
+      RbConfig::CONFIG['target_os'] =~ /mswin|mingw/i
+    end
+
+  end
+end

data/lib/loaders/excel_loader.rb

@@ -15,149 +15,149 @@ require 'datashift/exceptions'
 
 module DataShift
 
-  if(Guards::jruby?)
+  require 'loaders/loader_base'
 
-    require 'loaders/loader_base'
+  require 'excel'
 
-    require 'java'
-    require 'jexcel_file'
+  module ExcelLoading
 
-    module ExcelLoading
+    #  Options:
+    #   [:sheet_number]    : Default is 0. The index of the Excel Worksheet to use.
+    #   [:header_row]      : Default is 0. Use alternative row as header definition.
+    #   [:mandatory]       : Array of mandatory column names
+    #   [:force_inclusion] : Array of inbound column names to force into mapping
+    #   [:strict]          : Raise exception when no mapping found for a column heading (non mandatory)
 
-      #  Options:
-      #   [:sheet_number]    : Default is 0. The index of the Excel Worksheet to use.
-      #   [:header_row]      : Default is 0. Use alternative row as header definition.
-      #   [:mandatory]       : Array of mandatory column names
-      #   [:force_inclusion] : Array of inbound column names to force into mapping
-      #   [:strict]          : Raise exception when no mapping found for a column heading (non mandatory)
+    def perform_excel_load( file_name, options = {} )
 
+      raise MissingHeadersError, "Minimum row for Headers is 0 - passed #{options[:header_row]}" if(options[:header_row] && options[:header_row].to_i < 0)
+           
+      @excel = Excel.new
 
-      def perform_excel_load( file_name, options = {} )
-
-        @excel = JExcelFile.new
-
-        @excel.open(file_name)
+      @excel.open(file_name)
         
-        #if(options[:verbose])
-        puts "\n\n\nLoading from Excel file: #{file_name}"
+      #if(options[:verbose])
+      puts "\n\n\nLoading from Excel file: #{file_name}"
 
-        sheet_number = options[:sheet_number] || 0
+      sheet_number = options[:sheet_number] || 0
 
-        @sheet = @excel.sheet( sheet_number )
+      
+      @sheet = @excel.worksheet( sheet_number )
 
-        header_row_index =  options[:header_row] || 0
-        @header_row = @sheet.getRow(header_row_index)
+      header_row_index =  options[:header_row] || 0
+      @header_row = @sheet.row(header_row_index)
 
-        raise MissingHeadersError, "No headers found - Check Sheet #{@sheet} is complete and Row #{header_row_index} contains headers" unless(@header_row)
+      raise MissingHeadersError, "No headers found - Check Sheet #{@sheet} is complete and Row #{header_row_index} contains headers" unless(@header_row)
 
-        @headers = []
-
-        (0..JExcelFile::MAX_COLUMNS).each do |i|
-          cell = @header_row.getCell(i)
-          break unless cell
-          header = "#{@excel.cell_value(cell).to_s}".strip
-          break if header.empty?
-          @headers << header
-        end
+      @headers = []
 
-        raise MissingHeadersError, "No headers found - Check Sheet #{@sheet} is complete and Row #{header_row_index} contains headers" if(@headers.empty?)
-
-        # Create a method_mapper which maps list of headers into suitable calls on the Active Record class
-        # For example if model has an attribute 'price' will map columns called Price, price, PRICE etc to this attribute
-        map_headers_to_operators( @headers, options )
-
-        logger.info "Excel Loader processing #{@excel.num_rows} rows"
-        load_object_class.transaction do
-          @loaded_objects =  []
+      # TODO - make more robust - currently end on first empty column
+      # There is no actual max columns in Excel .. you will run out of memory though at some point
+      (0..1024).each do |column|
+        cell = @header_row[column]
+        break unless cell
+        header = "#{cell.to_s}".strip
+        break if header.empty?
+        @headers << header
+      end
 
-          (1..@excel.num_rows).collect do |row|
+      raise MissingHeadersError, "No headers found - Check Sheet #{@sheet} is complete and Row #{header_row_index} contains headers" if(@headers.empty?)
 
-            # Excel num_rows seems to return all 'visible' rows, which appears to be greater than the actual data rows
-            # (TODO - write spec to process .xls with a huge number of rows)
-            #
-            # This is rubbish but currently manually detect when actual data ends, this isn't very smart but
-            # got no better idea than ending once we hit the first completely empty row
-            break if @excel.sheet.getRow(row).nil?
+      
+      # Create a method_mapper which maps list of headers into suitable calls on the Active Record class
+      # For example if model has an attribute 'price' will map columns called Price, price, PRICE etc to this attribute
+      map_headers_to_operators( @headers, options )
 
-            contains_data = false
+      logger.info "Excel Loader processing #{@sheet.num_rows} rows"
+      
+      loaded_objects.clear
             
-            # First assign any default values for columns not included in parsed_file
-            process_missing_columns_with_defaults
-
+      load_object_class.transaction do
+       
+        @sheet.each_with_index do |row, i|
+                 
+          next if(i == header_row_index)
+          
+          # Excel num_rows seems to return all 'visible' rows, which appears to be greater than the actual data rows
+          # (TODO - write spec to process .xls with a huge number of rows)
+          #
+          # This is rubbish but currently manually detect when actual data ends, this isn't very smart but
+          # got no better idea than ending once we hit the first completely empty row
+          break if row.nil?
+
+          contains_data = false
+            
+          # First assign any default values for columns not included in parsed_file
+          process_missing_columns_with_defaults
 
-            # TODO - Smart sorting of column processing order ....
-            # Does not currently ensure mandatory columns (for valid?) processed first but model needs saving
-            # before associations can be processed so user should ensure mandatory columns are prior to associations
+          # TODO - Smart sorting of column processing order ....
+          # Does not currently ensure mandatory columns (for valid?) processed first but model needs saving
+          # before associations can be processed so user should ensure mandatory columns are prior to associations
 
-            # as part of this we also attempt to save early, for example before assigning to
-            # has_and_belongs_to associations which require the load_object has an id for the join table
+          # as part of this we also attempt to save early, for example before assigning to
+          # has_and_belongs_to associations which require the load_object has an id for the join table
          
-            # Iterate over the columns method_mapper found in Excel,
-            # pulling data out of associated column
-            @method_mapper.method_details.each_with_index do |method_detail, col|
+          # Iterate over the columns method_mapper found in Excel,
+          # pulling data out of associated column
+          @method_mapper.method_details.each_with_index do |method_detail, col|
 
-              value = value_at(row, col)
+            value = row[col]
 
-              contains_data = true unless(value.nil? || value.to_s.empty?)
+            contains_data = true unless(value.nil? || value.to_s.empty?)
               
-              prepare_data(method_detail, value)
+            prepare_data(method_detail, value)
               
-              process()
-            end
+            process()
+          end
                           
-            break unless(contains_data == true)
-
-            # TODO - requirements to handle not valid ?
-            # all or nothing or carry on and dump out the exception list at end
-            #puts "DEBUG: FINAL SAVE #{load_object.inspect}"
-            unless(save)
-              failure
-              logger.error "Failed to save row [#{row}]"
-              logger.error load_object.errors.inspect
-            else
-              logger.info "Row #{row} succesfully SAVED : ID #{load_object.id}"
-            end
+          break unless(contains_data == true)
+
+          # TODO - requirements to handle not valid ?
+          # all or nothing or carry on and dump out the exception list at end
+          #puts "DEBUG: FINAL SAVE #{load_object.inspect}"
+          unless(save)
+            failure
+            logger.error "Failed to save row [#{row}]"
+            logger.error load_object.errors.inspect
+          else
+            logger.info "Row #{row} succesfully SAVED : ID #{load_object.id}"
+          end
             
-            # don't forget to reset the object or we'll update rather than create
-            new_load_object
+          # don't forget to reset the object or we'll update rather than create
+          new_load_object
 
-          end
         end
-        puts "Excel loading stage complete - #{loaded_objects.size} rows added."
-        puts "There were NO failures." if failed_objects.empty?
-        
-        puts "WARNING : Check logs : #{failed_objects.size} rows contained errors and #{failed_objects.size} records NOT created." unless failed_objects.empty?
       end
+      
+      loaded_objects.compact! if(loaded_objects)
+      
+      puts "Excel loading stage complete - #{loaded_objects.size} rows added."
+      puts "There were NO failures." if failed_objects.empty?
+        
+      puts "WARNING : Check logs : #{failed_objects.size} rows contained errors and #{failed_objects.size} records NOT created." unless failed_objects.empty?
+    end
 
-      def value_at(row, column)
-        @excel.get_cell_value( @excel.sheet.getRow(row), column)
-      end
+    def value_at(row, column)
+      @excel[row, column]
     end
+  end
 
 
-    class ExcelLoader < LoaderBase
+  class ExcelLoader < LoaderBase
 
-      include ExcelLoading
+    include ExcelLoading
   
-      def initialize(klass, object = nil, options = {})
-        super( klass, object, options )
-        raise "Cannot load - failed to create a #{klass}" unless @load_object
-      end
-
+    def initialize(klass, object = nil, options = {})
+      super( klass, object, options )
+      raise "Cannot load - failed to create a #{klass}" unless @load_object
+    end
 
-      def perform_load( file_name, options = {} )
-        perform_excel_load( file_name, options )
 
-        puts "Excel loading stage complete - #{loaded_objects.size} rows added."  
-      end
+    def perform_load( file_name, options = {} )
+      perform_excel_load( file_name, options )
 
+      puts "Excel loading stage complete - #{loaded_objects.size} rows added."  
     end
-  
-  else
-  
-    module ExcelLoading 
-    end
-  
+
   end
-  
 end

data/lib/loaders/loader_base.rb

@@ -28,8 +28,10 @@ module DataShift
 
     attr_accessor :loaded_objects, :failed_objects
 
-    attr_accessor :options, :verbose
+    attr_accessor :config, :verbose
 
+    def options() return @config; end
+    
     # Support multiple associations being added to a base object to be specified in a single column.
     # 
     # Entry represents the association to find via supplied name, value to use in the lookup.
@@ -110,9 +112,9 @@ module DataShift
       end unless(options[:load] == false)
       
       @method_mapper = DataShift::MethodMapper.new
-      @options = options.dup    # clone can cause issues like 'can't modify frozen hash'
+      @config = options.dup    # clone can cause issues like 'can't modify frozen hash'
 
-      @verbose = @options[:verbose]
+      @verbose = @config[:verbose]
       @headers = []
 
       @default_data_objects ||= {}
@@ -123,6 +125,8 @@ module DataShift
       @prefixes       = {}
       @postfixes      = {}
       
+      @loaded_objects = []
+      
       reset(object)
     end
 
@@ -149,7 +153,6 @@ module DataShift
       ext = File.extname(file_name)
           
       if(ext.casecmp('.xls') == 0)
-        raise DataShift::BadRuby, "Please install and use JRuby for loading .xls files" unless(Guards::jruby?)
         perform_excel_load(file_name, options)
       elsif(ext.casecmp('.csv') == 0)
         perform_csv_load(file_name, options)
@@ -227,23 +230,42 @@ module DataShift
     end
     
     
-    # Find a record for model klazz, looking up on field for x
+    # Find a record for model klazz, looking up on field containing search_terms
     # Responds to global Options :
-    # :case_sensitive : Default is a case insensitive lookup.
-    # :use_like : Attempts a lookup using ike and x% ratehr than equality
+    #   :case_sensitive : Default is a case insensitive lookup.
+    #   :use_like : Attempts a lookup using ike and x% ratehr than equality 
     
-    def get_record_by(klazz, field, x)
+    def get_record_by(klazz, field, search_terms, split_on = ' ', split_on_prefix = nil)
     
       begin
-        if(@options[:case_sensitive]) 
-          return klazz.send("find_by_#{field}", x)
-        elsif(@options[:use_like])
-          return klazz.where("#{field} like ?", "#{x}%").first
+        record = if(@config[:case_sensitive]) 
+          klazz.send("find_by_#{field}", search_terms)
+        elsif(@config[:use_like])
+          klazz.where("#{field} like ?", "#{search_terms}%").first
         else
-          return klazz.where("lower(#{field}) = ?", x.downcase).first
+          klazz.where("lower(#{field}) = ?", search_terms.downcase).first
         end
+    
+        # try the separate individual portions of the search_terms, front -> back
+        search_terms.split(split_on).each do |x|
+          z = "#{split_on_prefix}#{x}" if(split_on_prefix)
+         
+          record = get_record_by(klazz, field, z, split_on, split_on_prefix)
+          break if record
+        end unless(record)
+            
+        # this time try sequentially and incrementally scanning
+        search_terms.split(split_on).inject("") do |str, term|
+          z = (split_on_prefix) ? "#{split_on_prefix}#{str}#{term}": "#{str}#{term}"
+          record = get_record_by(klazz, field, z, split_on, split_on_prefix)
+          break if record
+          term
+        end unless(record)      
+                
+        return record 
+        
       rescue => e
-        logger.error("Exception attempting to find a record for [#{x}] on #{klazz}.#{field}")
+        logger.error("Exception attempting to find a record for [#{search_terms}] on #{klazz}.#{field}")
         logger.error e.backtrace
         logger.error e.inspect
         return nil
@@ -311,14 +333,14 @@ module DataShift
       end
       
       if(data['LoaderBase'])
-        @options.merge!(data['LoaderBase'])
+        @config.merge!(data['LoaderBase'])
       end
        
       if(data[self.class.name])    
-        @options.merge!(data[self.class.name])
+        @config.merge!(data[self.class.name])
       end
       
-      logger.info("Loader Options : #{@options.inspect}")
+      logger.info("Loader Options : #{@config.inspect}")
     end
     
     # Set member variables to hold details and value.
@@ -380,7 +402,7 @@ module DataShift
     #
     def process()  
       
-      logger.info("Current value to assign : #{@current_value}") #if @options['verboose_logging']
+      logger.info("Current value to assign : #{@current_value}") #if @config['verboose_logging']
       
       if(@current_method_detail.operator_for(:has_many))
 
@@ -420,7 +442,7 @@ module DataShift
               unless(find_by_values.size == @current_value.size)
                 found = @current_value.collect {|f| f.send(find_operator) }
                 @load_object.errors.add( @current_method_detail.operator, "Association with key(s) #{(find_by_values - found).inspect} NOT found")
-                puts "WARNING: Association with key(s) #{(lookups - found).inspect} NOT found - Not added."
+                puts "WARNING: Association #{@current_method_detail.operator} with key(s) #{(find_by_values - found).inspect} NOT found - Not added."
                 next if(@current_value.empty?)
               end
 
@@ -464,7 +486,7 @@ module DataShift
         failure
         puts "Error saving #{@load_object.class} : #{e.inspect}"
         logger.error e.backtrace
-        raise "Error in save whilst processing column #{@current_method_detail.name}" if(@options[:strict])
+        raise "Error in save whilst processing column #{@current_method_detail.name}" if(@config[:strict])
       end
     end
 
@@ -522,7 +544,7 @@ module DataShift
     end
 
     def abort_on_failure?
-      @options[:abort_on_failure] == 'true'
+      @config[:abort_on_failure] == 'true'
     end
 
     def loaded_count