ar_loader 0.0.6 → 0.0.8

data/lib/loaders/excel_loader.rb

@@ -0,0 +1,132 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2011
+# License::   MIT
+#
+# Details::   Specific loader to support Excel files.
+#             Note this only requires JRuby, Excel not required, nor Win OLE
+#
+require 'ar_loader/exceptions'
+
+if(Guards::jruby?)
+
+
+  require 'loaders/loader_base'
+  require 'ar_loader/method_mapper'
+
+  require 'java'
+  require 'jexcel_file'
+
+  module ARLoader
+     
+    class ExcelLoader < LoaderBase
+  
+      def initialize(klass, object = nil, options = {})
+        super( klass, object, options )
+        raise "Cannot load - failed to create a #{klass}" unless @load_object
+      end
+      
+
+      def load( file_name, options = {} )
+
+        @mandatory = options[:mandatory] || nil
+
+        @excel = JExcelFile.new
+
+        @excel.open(file_name)
+
+        sheet_number = options[:sheet_number] || 0
+
+        @sheet = @excel.sheet( sheet_number )
+
+        header_row =  options[:header_row] || 0
+        @header_row = @sheet.getRow(header_row)
+
+        raise "ERROR: No headers found - Check Sheet #{@sheet} is completed sheet and Row 1 contains headers" unless(@header_row)
+
+        @headers = []
+        (0..JExcelFile::MAX_COLUMNS).each do |i|
+          cell = @header_row.getCell(i)
+          break unless cell
+          header = "#{@excel.cell_value(cell).to_s}".strip
+          break if header.empty?
+          @headers << header
+        end
+
+        raise "ERROR: No headers found - Check Sheet #{@sheet} is completed sheet and Row 1 contains headers" if(@headers.empty?)
+
+
+        @method_mapper = ARLoader::MethodMapper.new
+
+        # Convert the list of headers into suitable calls on the Active Record class
+        @method_mapper.populate_methods( load_object_class, @headers )
+
+        unless(@method_mapper.missing_methods.empty?)
+          puts "WARNING: Following column headings could not be mapped : #{@method_mapper.missing_methods.inspect}"
+          raise MappingDefinitionError, "ERROR: Missing mappings for #{@method_mapper.missing_methods.size} column headings"
+        end
+
+        unless(@method_mapper.contains_mandatory?( options[:mandatory]) )
+          missing_mandatory( options[:mandatory]).each { |e| puts "ERROR: Mandatory column missing - need a '#{e}' column" }
+          raise "Bad File Description - Mandatory columns missing  - please fix and retry."
+        end if(options[:mandatory])
+
+        #if(options[:verbose])
+        puts "\n\n\nLoading from Excel file: #{file_name}"
+
+        load_object_class.transaction do
+          @loaded_objects =  []
+
+          (1..@excel.num_rows).collect do |row|
+
+            # Excel num_rows seems to return all 'visible' rows, which appears to be greater than the actual data rows
+            # (TODO - write spec to process .xls with a huge number of rows)
+            #
+            # So currently we have to manually detect when actual data ends, this isn't very smart but
+            # currently got no better idea than ending once we hit the first completely empty row
+            break if @excel.sheet.getRow(row).nil?
+
+            contains_data = false
+
+            # TODO - Smart sorting of column processing order ....
+            # Does not currently ensure mandatory columns (for valid?) processed first but model needs saving
+            # before associations can be processed so user should ensure mandatory columns are prior to associations
+
+            # as part of this we also attempt to save early, for example before assigning to
+            # has_and_belongs_to associations which require the load_object has an id for the join table
+
+            # Iterate over the columns method_mapper found in Excel,
+            # pulling data out of associated column
+            @method_mapper.method_details.each_with_index do |method_detail, col|
+
+              value = value_at(row, col)
+
+              contains_data = true unless(value.nil? || value.to_s.empty?)
+
+              #puts "DEBUG: Excel process METHOD :#{method_detail.inspect}"
+              #puts "DEBUG: Excel process VALUE  :#{value.inspect}"
+              process(method_detail, value)
+            end
+
+            break unless contains_data
+
+            # TODO - requirements to handle not valid ?
+            # all or nothing or carry on and dump out the exception list at end
+
+            save
+
+            # don't forget to reset the object or we'll update rather than create
+            new_load_object
+        
+          end
+        end
+        puts "Excel loading stage complete - #{loaded_objects.size} rows added."
+      end
+
+      def value_at(row, column)
+        @excel.value( @excel.sheet.getRow(row), column)
+      end
+    
+    end
+  end
+end

data/lib/loaders/loader_base.rb

@@ -1,74 +1,206 @@
-# Copyright:: (c) Autotelik Media Ltd 2011
-# Author ::   Tom Statter
-# Date ::     Aug 2010
-# License::   MIT
-#
-#  Details::  Base class for loaders, providing a process hook which populates a model,
-#             based on a method map and supplied value from a file - i.e a single column/row's string value.
-#             Note that although a single column, the string can be formatted to contain multiple values.
-#
-#             Tightly coupled with MethodMapper classes (in lib/engine) which contains full details of
-#             a file's column and it's correlated AR associations.
-#
-class LoaderBase
-
-  attr_accessor :load_object_class, :load_object, :value
-
-  # Enable single column (association) to contain multiple name/value sets in default form :
-  #   Name1:value1, value2|Name2:value1, value2, value3|Name3:value1, value2
-  #
-  # E.G.
-  #   Row for association could have a name (Size/Colour/Sex) with a set of values,
-  #   and this combination can be expressed multiple times :
-  #   Size:small,medium,large|Colour:red, green|Sex:Female
-
-  @@name_value_delim  = ':'
-  @@multi_value_delim = ','
-  @@multi_assoc_delim = '|'
-
-  def self.set_name_value_delim(x)  @@name_value_delim = x; end
-  def self.set_multi_value_delim(x) @@multi_value_delim = x; end
-  def self.set_multi_assoc_delim(x) @@multi_assoc_delim = x; end
-
-  def initialize(object_class, object = nil)
-    @load_object_class = object_class
-    @load_object = object || @load_object_class.new
-  end
-
-  def reset()
-    @load_object = @load_object_class.new
-  end
-  
-  # What process a value string from a column.
-  # Assigning value(s) to correct association on @load_object.
-  # Method map represents a column from a file and it's correlated AR associations.
-  # Value string which may contain multiple values for a collection association.
-  # 
-  def process(method_map, value)
-    #puts "INFO: LOADER BASE processing #{@load_object}"
-    @value = value
-    
-    if(method_map.has_many && method_map.has_many_class && @value)
-      # The Generic handler for Associations
-      # The actual class of the association so we can find_or_create on it
-      assoc_class = method_map.has_many_class
-
-      puts "Processing Association: #{assoc_class} : #{@value}"
-
-      @value.split(@@multi_assoc_delim).collect do |lookup|
-        # TODO - Don't just rely on 'name' but try different finds as per MethodMappe::insistent_belongs_to ..
-        x = assoc_class.find(:first, :conditions => ['lower(name) LIKE ?', "%#{lookup.downcase}%"])
-        unless x
-          puts "WARNING: #{lookup} in #{assoc_class} NOT found - Not added to #{@load_object.class}"
-          next
-        end
-        @load_object.send( method_map.has_many ) << x
-        @load_object.save
-      end
-    else
-      # Nice n simple straight assignment to a column variable
-      method_map.assign(@load_object, @value) unless method_map.has_many
-    end
-  end
-
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2010
+# License::   MIT
+#
+#  Details::  Base class for loaders, providing a process hook which populates a model,
+#             based on a method map and supplied value from a file - i.e a single column/row's string value.
+#             Note that although a single column, the string can be formatted to contain multiple values.
+#
+#             Tightly coupled with MethodMapper classes (in lib/engine) which contains full details of
+#             a file's column and it's correlated AR associations.
+#
+module ARLoader
+
+  class LoaderBase
+
+    attr_reader :headers
+
+    attr_accessor :load_object_class, :load_object
+    attr_accessor :current_value, :current_method_detail
+
+    attr_accessor :loaded_objects, :failed_objects
+
+    attr_accessor :options
+
+    # Enable an entry representing an association to contain multiple lookup name/value sets in default form :
+    #   Name1:value1, value2|Name2:value1, value2, value3|Name3:value1, value2
+    #
+    # E.G.
+    #   Row for association could have a columns called Size and once called Colour,
+    #   and this combination could be used to lookup multiple associations to add to the main model
+    #
+    #       Size:small            # => generates find_by_size( 'small' )
+    #       Size:large|           # => generates find_by_size( 'large' )
+    #       Colour:red,green,blue # => generates find_all_by_colour( ['red','green','blue'] )
+    #
+    # TODO - support embedded object creation/update via hash (which hopefully we should be able to just forward to AR)
+    #
+    #      |Category|
+    #      name:new{ :date => '20110102', :owner = > 'blah'}
+    #
+    @@name_value_delim  = ':'
+    @@multi_value_delim = ','
+
+    # TODO - support multi embedded object creation/update via hash (which hopefully we should be able to just forward to AR)
+    #
+    #      |Category|
+    #      name:new{ :a => 1, :b => 2}|name:medium{ :a => 6, :b => 34}|name:old{ :a => 12, :b => 67}
+    #
+    @@multi_assoc_delim = '|'     # Used to delimit multiple entries in a single column
+
+    def self.set_name_value_delim(x)  @@name_value_delim = x; end
+    def self.set_multi_value_delim(x) @@multi_value_delim = x; end
+    def self.set_multi_assoc_delim(x) @@multi_assoc_delim = x; end
+
+    # Options
+    def initialize(object_class, object = nil, options = {})
+      @load_object_class = object_class
+
+      # Gather list of all possible 'setter' methods on AR class (instance variables and associations)
+      ARLoader::MethodMapper.find_operators( @load_object_class )
+
+      @options = options.clone
+      @headers = []
+      reset(object)
+    end
+
+    def reset(object = nil)
+      @load_object = object || new_load_object
+      @loaded_objects, @failed_objects = [],[]
+      @current_value = nil
+    end
+
+    def contains_mandatory?( mandatory_list )
+      puts "HEADERS", @headers
+      [mandatory_list - @headers].flatten.empty?
+    end
+
+    def missing_mandatory( mandatory_list )
+      [mandatory_list - @headers].flatten
+    end
+
+    def new_load_object
+      @load_object = @load_object_class.new
+      @load_object
+    end
+
+    def abort_on_failure?
+      @options[:abort_on_failure] == 'true'
+    end
+
+    def loaded_count
+      @loaded_objects.size
+    end
+
+    def failed_count
+      @failed_objects.size
+    end
+  
+    # Search method mapper for supplied klass and column,
+    # and if suitable association found, process row data into current load_object
+    def find_and_process(klass, column_name, row_data)
+      method_detail = MethodMapper.find_method_detail( klass, column_name )
+
+   
+      if(method_detail)
+        process(method_detail, row_data)
+      else
+        @load_object.errors.add_base( "No matching method found for column #{column_name}")
+      end
+    end
+
+
+    # kinda the derived classes interface - best way in Ruby ?
+    def load( input, options = {} )
+      raise "WARNING- ABSTRACT METHOD CALLED - Please implement load()"
+    end
+
+
+    # What process a value string from a column.
+    # Assigning value(s) to correct association on @load_object.
+    # Method detail represents a column from a file and it's correlated AR associations.
+    # Value string which may contain multiple values for a collection association.
+    #
+    def process(method_detail, value)
+      #puts "INFO: LOADER BASE processing #{@load_object}"
+      @current_value = value
+      @current_method_detail = method_detail
+
+      if(method_detail.operator_for(:has_many))
+
+        if(method_detail.operator_class && @current_value)
+
+          # there are times when we need to save early, for example before assigning to
+          # has_and_belongs_to associations which require the load_object has an id for the join table
+        
+          save if( load_object.valid? && load_object.new_record? )
+
+          # A single column can contain multiple associations delimited by special char
+          columns = @current_value.split(@@multi_assoc_delim)
+
+          # Size:large|Colour:red,green,blue   => generates find_by_size( 'large' ) and find_all_by_colour( ['red','green','blue'] )
+
+          columns.each do |assoc|
+            operator, values = assoc.split(@@name_value_delim)
+
+            lookups = values.split(@@multi_value_delim)
+
+            if(lookups.size > 1)
+
+              @current_value = method_detail.operator_class.send("find_all_by_#{operator}", lookups )
+
+              unless(lookups.size == @current_value.size)
+                found = @current_value.collect {|f| f.send(operator) }
+                @load_object.errors.add( method_detail.operator, "Association with key(s) #{(lookups - found).inspect} NOT found")
+                puts "WARNING: Association with key(s) #{(lookups - found).inspect} NOT found - Not added."
+                next if(@current_value.empty?)
+              end
+
+            else
+
+              @current_value = method_detail.operator_class.send("find_by_#{operator}", lookups )
+
+              unless(@current_value)
+                @load_object.errors.add( method_detail.operator, "Association with key #{lookups} NOT found")
+                puts "WARNING: Association with key #{lookups} NOT found - Not added."
+                next
+              end
+
+            end
+
+            # Lookup Assoc's Model done, now add the found value(s) to load model's collection
+            method_detail.assign(@load_object, @current_value)
+          end
+        end
+        # END HAS_MANY
+      else
+        # Nice n simple straight assignment to a column variable
+        method_detail.assign(@load_object, @current_value)
+      end
+    end
+
+    def save
+      puts "SAVING #{load_object.class} : #{load_object.inspect}" #if(options[:verbose])
+      begin
+        @load_object.save
+        @loaded_objects << load_object unless(@loaded_objects.include?(load_object))
+      rescue => e
+        @failed_objects << load_object unless(@failed_objects.include?(load_object))
+        puts "Error saving #{load_object.class} : #{e.inspect}"
+        raise "Error in save whilst processing column #{@current_method_detail.name}" if(@options[:strict])
+      end
+    end
+
+    def find_or_new( klass, condition_hash = {} )
+      @records[klass] = klass.find(:all, :conditions => condition_hash)
+      if @records[klass].any?
+        return @records[klass].first
+      else
+        return klass.new
+      end
+    end
+  
+  end
+
 end

data/lib/loaders/spree/image_loader.rb

@@ -1,42 +1,46 @@
-# Copyright:: (c) Autotelik Media Ltd 2011
-# Author ::   Tom Statter
-# Date ::     Jan 2011
-# License::   MIT. Free, Open Source.
-#
-require 'loader_base'
-
-class ImageLoader < LoaderBase
-
-  def initialize(klass = Image, image = nil)
-    super( klass, image )
-    raise "Failed to create Image for loading" unless @load_object
-  end
-
-  # Note the Spree Image model sets default storage path to
-  # => :path => ":rails_root/public/assets/products/:id/:style/:basename.:extension"
-
-  def process( image_path, record = nil)
-
-    unless File.exists?(image_path)
-      puts "ERROR : Invalid Path"
-      return
-    end
-
-    alt = (record and record.respond_to? :name) ? record.name : ""
-
-    @load_object.alt = alt
-
-    begin
-      @load_object.attachment = File.new(image_path, "r")
-    rescue => e
-      puts e.inspect
-      puts "ERROR : Failed to read image #{image_path}"
-      return
-    end
-
-    @load_object.attachment.reprocess!
-    @load_object.viewable = record if record
-
-    puts @load_object.save ? "Success: Uploaded Image: #{@load_object.inspect}" : "ERROR : Problem saving to DB Image: #{@load_object}"
-  end
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Jan 2011
+# License::   MIT. Free, Open Source.
+#
+require 'loader_base'
+
+module ARLoader
+
+  class ImageLoader < LoaderBase
+
+    def initialize(image = nil)
+      super( Image, image )
+      raise "Failed to create Image for loading" unless @load_object
+    end
+
+    # Note the Spree Image model sets default storage path to
+    # => :path => ":rails_root/public/assets/products/:id/:style/:basename.:extension"
+
+    def process( image_path, record = nil)
+
+      unless File.exists?(image_path)
+        puts "ERROR : Invalid Path"
+        return
+      end
+
+      alt = (record and record.respond_to? :name) ? record.name : ""
+
+      @load_object.alt = alt
+
+      begin
+        @load_object.attachment = File.new(image_path, "r")
+      rescue => e
+        puts e.inspect
+        puts "ERROR : Failed to read image #{image_path}"
+        return
+      end
+
+      @load_object.attachment.reprocess!
+      @load_object.viewable = record if record
+
+      puts @load_object.save ? "Success: Uploaded Image: #{@load_object.inspect}" : "ERROR : Problem saving to DB Image: #{@load_object}"
+    end
+  end
+
 end