datashift 0.0.1

data/lib/loaders/spreadsheet_loader.rb

@@ -0,0 +1,137 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Jan 2012
+# License::   MIT
+#
+# Details::   Specific loader to support Excel files via http://rubygems.org/gems/spreadsheet
+#             
+#             Offers an alternative for non JRuby usage(see excel_loader)
+#
+#             Maps column headings to operations on the model.
+#             Iterates over all the rows using mapped operations to assign row data to a database object,
+#             i.e pulls data from each column and sends to object.
+#
+require 'datashift/exceptions'
+
+module DataShift
+
+  require 'loaders/loader_base'
+
+  module SpreadsheetLoading
+
+    #  Options:
+    #   [:header_row]   : Default is 0. Use alternative row as header definition.
+    #   [:mandatory]    : Array of mandatory column names
+    #   [:strict]       : Raise exception when no mapping found for a column heading (non mandatory)
+    #   [:sheet_number]
+
+    def perform_spreadsheet_load( file_name, options = {} )
+
+      @mandatory = options[:mandatory] || []
+
+      @excel = JExcelFile.new
+
+      @excel.open(file_name)
+        
+      #if(options[:verbose])
+      puts "\n\n\nLoading from Excel file: #{file_name}"
+
+      sheet_number = options[:sheet_number] || 0
+
+      @sheet = @excel.sheet( sheet_number )
+
+      header_row_index =  options[:header_row] || 0
+      @header_row = @sheet.getRow(header_row_index)
+
+      raise MissingHeadersError, "No headers found - Check Sheet #{@sheet} is complete and Row #{header_row_index} contains headers" unless(@header_row)
+
+      @headers = []
+        
+      (0..JExcelFile::MAX_COLUMNS).each do |i|
+        cell = @header_row.getCell(i)
+        break unless cell
+        header = "#{@excel.cell_value(cell).to_s}".strip
+        break if header.empty?
+        @headers << header
+      end
+
+      raise MissingHeadersError, "No headers found - Check Sheet #{@sheet} is complete and Row #{header_row_index} contains headers" if(@headers.empty?)
+
+      # Create a method_mapper which maps list of headers into suitable calls on the Active Record class
+      map_headers_to_operators( @headers, options[:strict] , @mandatory )
+
+      load_object_class.transaction do
+        @loaded_objects =  []
+
+        (1..@excel.num_rows).collect do |row|
+
+          # Excel num_rows seems to return all 'visible' rows, which appears to be greater than the actual data rows
+          # (TODO - write spec to process .xls with a huge number of rows)
+          #
+          # This is rubbish but currently manually detect when actual data ends, this isn't very smart but
+          # got no better idea than ending once we hit the first completely empty row
+          break if @excel.sheet.getRow(row).nil?
+
+          contains_data = false
+
+          # TODO - Smart sorting of column processing order ....
+          # Does not currently ensure mandatory columns (for valid?) processed first but model needs saving
+          # before associations can be processed so user should ensure mandatory columns are prior to associations
+
+          # as part of this we also attempt to save early, for example before assigning to
+          # has_and_belongs_to associations which require the load_object has an id for the join table
+
+          # Iterate over the columns method_mapper found in Excel,
+          # pulling data out of associated column
+          @method_mapper.method_details.each_with_index do |method_detail, col|
+
+            value = value_at(row, col)
+
+            contains_data = true unless(value.nil? || value.to_s.empty?)
+
+            #puts "DEBUG: Excel process METHOD :#{method_detail.inspect}", value.inspect
+            prepare_data(method_detail, value)
+              
+            process()
+          end
+                          
+          break unless(contains_data == true)
+
+          # TODO - requirements to handle not valid ?
+          # all or nothing or carry on and dump out the exception list at end
+          #puts "DEBUG: FINAL SAVE #{load_object.inspect}"
+          save
+          #puts "DEBUG: SAVED #{load_object.inspect}"
+
+          # don't forget to reset the object or we'll update rather than create
+          new_load_object
+
+        end
+      end
+      puts "Excel loading stage complete - #{loaded_objects.size} rows added."
+    end
+
+    def value_at(row, column)
+      @excel.get_cell_value( @excel.sheet.getRow(row), column)
+    end
+  end
+
+
+  class SpreadsheetLoader < LoaderBase
+
+    include SpreadsheetLoading
+  
+    def initialize(klass, object = nil, options = {})
+      super( klass, object, options )
+      raise "Cannot load - failed to create a #{klass}" unless @load_object
+    end
+
+    def perform_load( file_name, options = {} )
+      perform_spreadsheet_load( file_name, options )
+
+      puts "Spreadsheet loading stage complete - #{loaded_objects.size} rows added."  
+    end
+
+  end
+  
+end

data/lib/loaders/spree/image_loader.rb

@@ -0,0 +1,46 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Jan 2011
+# License::   MIT. Free, Open Source.
+#
+require 'loader_base'
+
+module DataShift
+
+  class ImageLoader < LoaderBase
+
+    def initialize(image = nil)
+      super( Image, image )
+      raise "Failed to create Image for loading" unless @load_object
+    end
+
+    # Note the Spree Image model sets default storage path to
+    # => :path => ":rails_root/public/assets/products/:id/:style/:basename.:extension"
+
+    def process( image_path, record = nil)
+
+      unless File.exists?(image_path)
+        puts "ERROR : Invalid Path"
+        return
+      end
+
+      alt = (record and record.respond_to? :name) ? record.name : ""
+
+      @load_object.alt = alt
+
+      begin
+        @load_object.attachment = File.new(image_path, "r")
+      rescue => e
+        puts e.inspect
+        puts "ERROR : Failed to read image #{image_path}"
+        return
+      end
+
+      @load_object.attachment.reprocess!
+      @load_object.viewable = record if record
+
+      puts @load_object.save ? "Success: Uploaded Image: #{@load_object.inspect}" : "ERROR : Problem saving to DB Image: #{@load_object}"
+    end
+  end
+
+end

data/lib/loaders/spree/product_loader.rb

@@ -0,0 +1,225 @@
+# Copyright:: (c) Autotelik Media Ltd 2010
+# Author ::   Tom Statter
+# Date ::     Aug 2010
+# License::   MIT ?
+#
+# Details::   Specific over-rides/additions to support Spree Products
+#
+require 'loader_base'
+require 'excel_loader'
+require 'csv_loader'
+
+module DataShift
+
+  module Spree
+
+    class ProductLoader < LoaderBase
+
+      include DataShift::CsvLoading
+      include DataShift::ExcelLoading
+
+      def initialize(product = nil)
+        super( Product, product, :instance_methods => true  )
+        raise "Failed to create Product for loading" unless @load_object
+      end
+
+
+      # Based on filename call appropriate loading function
+      # Currently supports :
+      #   Excel/Open Office files saved as .xls
+      #   CSV files
+      def perform_load( file_name, options = {} )
+
+        ext = File.extname(file_name)
+          
+        if(ext == '.xls')
+          raise DataShift::BadRuby, "Please install and use JRuby for loading .xls files" unless(Guards::jruby?)
+          perform_excel_load(file_name, options)
+        elsif(ext == '.csv')
+          perform_csv_load(file_name, options)
+        else
+          raise DataShift::UnsupportedFileType, "#{ext} files not supported - Try CSV or OpenOffice/Excel .xls"
+        end
+      end
+      
+      # Over ride base class process with some Spree::Product specifics
+      #
+      # What process a value string from a column, assigning value(s) to correct association on Product.
+      # Method map represents a column from a file and it's correlated Product association.
+      # Value string which may contain multiple values for a collection (has_many) association.
+      #
+      def process()
+
+        # Special cases for Products, generally where a simple one stage lookup won't suffice
+        # otherwise simply use default processing from base class
+        if((@current_method_detail.operator?('variants') || @current_method_detail.operator?('option_types')) && current_value)
+
+          add_options
+
+        elsif(@current_method_detail.operator?('taxons') && current_value)
+
+          add_taxons
+
+        elsif(@current_method_detail.operator?('product_properties') && current_value)
+
+          add_properties
+          
+        elsif(@current_method_detail.operator?('count_on_hand') || @current_method_detail.operator?('on_hand') )
+
+          # Unless we can save here, in danger of count_on_hand getting wiped out.
+          # If we set (on_hand or count_on_hand) on an unsaved object, during next subsequent save
+          # looks like some validation code or something calls Variant.on_hand= with 0
+          # If we save first, then our values seem to stick
+
+          # TODO smart column ordering to ensure always valid - if we always make it very last column might not get wiped ?
+          # 
+          save_if_new
+         
+          # Spree has some stock management stuff going on, so dont usually assign to column vut use
+          # on_hand and on_hand=
+          if(@load_object.variants.size > 0 && current_value.include?(LoaderBase::multi_assoc_delim))
+
+            #puts "DEBUG: COUNT_ON_HAND PER VARIANT",current_value.is_a?(String),
+              #&& current_value.is_a?(String) && current_value.include?(LoaderBase::multi_assoc_delim))
+            # Check if we processed Option Types and assign count per option
+            values = current_value.to_s.split(LoaderBase::multi_assoc_delim)
+
+            if(@load_object.variants.size == values.size)
+              @load_object.variants.each_with_index {|v, i| v.on_hand = values[i]; v.save; }
+            else
+              puts "WARNING: Count on hand entries did not match number of Variants - None Set"
+            end
+          else
+            #puts "DEBUG: COUNT_ON_HAND #{current_value.to_i}"
+            load_object.on_hand = current_value.to_i
+          end
+
+        else
+          super
+        end
+      end
+
+      private
+
+      # Special case for OptionTypes as it's two stage process
+      # First add the possible option_types to Product, then we are able
+      # to define Variants on those options values.
+      #
+      def add_options
+        # TODO smart column ordering to ensure always valid by time we get to associations
+        save_if_new
+
+        option_types = current_value.split( LoaderBase::multi_assoc_delim )
+
+        option_types.each do |ostr|
+          oname, value_str = ostr.split(LoaderBase::name_value_delim)
+
+          option_type = OptionType.find_by_name(oname)
+
+          unless option_type
+            option_type = OptionType.create( :name => oname, :presentation => oname.humanize)
+            # TODO - dynamic creation should be an option
+
+            unless option_type
+              puts "WARNING: OptionType #{oname} NOT found - Not set Product"
+              next
+            end
+          end
+
+          @load_object.option_types << option_type unless @load_object.option_types.include?(option_type)
+
+          # Can be simply list of OptionTypes, some or all without values
+          next unless(value_str)
+
+          # Now get the value(s) for the option e.g red,blue,green for OptType 'colour'
+          ovalues = value_str.split(',')
+
+          ovalues.each_with_index do |ovname, i|
+            ovname.strip!
+            ov = OptionValue.find_or_create_by_name(ovname)
+            if ov
+              object = Variant.create( :product => @load_object, :sku => "#{@load_object.sku}_#{i}", :price => @load_object.price, :available_on => @load_object.available_on)
+              #puts "DEBUG: Create New Variant: #{object.inspect}"
+              object.option_values << ov
+              #@load_object.variants << object
+            else
+              puts "WARNING: Option #{ovname} NOT FOUND - No Variant created"
+            end
+          end
+        end
+
+      end
+
+
+      # Special case for ProductProperties since it can have additional value applied.
+      # A list of Properties with a optional Value - supplied in form :
+      #   property.name:value|property.name|property.name:value
+      #
+      def add_properties
+        # TODO smart column ordering to ensure always valid by time we get to associations
+        save_if_new
+
+        property_list = current_value.split(LoaderBase::multi_assoc_delim)
+
+        property_list.each do |pstr|
+          pname, pvalue = pstr.split(LoaderBase::name_value_delim)
+          property = Property.find_by_name(pname)
+
+          unless property
+            property = Property.create( :name => pname, :presentation => pname.humanize)
+          end
+
+          if(property)
+            @load_object.product_properties << ProductProperty.create( :property => property, :value => pvalue)
+          else
+            puts "WARNING: Property #{pname} NOT found - Not set Product"
+          end
+         
+        end
+      
+      end
+
+      
+      def add_taxons
+        # TODO smart column ordering to ensure always valid by time we get to associations
+        save_if_new
+
+        name_list = current_value.split(LoaderBase::multi_assoc_delim)
+
+        taxons = name_list.collect do |t|
+
+          taxon = Taxon.find_by_name(t)
+
+          unless taxon
+            parent = Taxonomy.find_by_name(t)
+
+            begin
+              if(parent)
+                # not sure this can happen but just incase we get a weird situation where we have
+                # a taxonomy without a root named the same - create the child taxon we require
+                taxon = Taxon.create(:name => t, :taxonomy_id => parent.id)
+              else
+                parent = Taxonomy.create!( :name => t )
+
+                taxon = parent.root
+              end
+
+            rescue => e
+              e.backtrace
+              e.inspect
+              puts "ERROR : Cannot assign Taxon ['#{t}'] to Product ['#{load_object.name}']"
+              next
+            end
+          end
+          taxon
+        end
+
+        taxons.compact!
+
+        @load_object.taxons << taxons unless(taxons.empty?)
+
+      end
+
+    end
+  end
+end

data/spec/csv_loader_spec.rb

@@ -0,0 +1,31 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2011
+# License::   MIT
+#
+# Details::   Specs for Excel aspect of Active Record Loader
+#
+require File.dirname(__FILE__) + '/spec_helper'
+
+require 'erb'
+require 'excel_loader'
+
+describe 'CSV Loader' do
+
+  before(:all) do
+    
+    # load our test model definitions - Project etc
+    require File.join($DataShiftFixturePath, 'test_model_defs')  
+       
+    db_connect( 'test_file' )    # , test_memory, test_mysql
+    migrate_up
+    @klazz = Project
+  end
+  
+  before(:each) do
+    MethodMapper.clear
+    MethodMapper.find_operators( @klazz )
+    MethodMapper.find_operators( @assoc_klazz )
+  end
+
+end

data/spec/datashift_spec.rb

@@ -0,0 +1,27 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Oct 2011
+# License::   MIT
+#
+# Details::   Specs for high level apsects of DataShift library
+#
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe 'DataShift' do
+
+  before(:each) do
+  end
+  
+  it "should provide gem version" do
+    DataShift::gem_version
+  end
+
+  it "should provide gem name" do
+    DataShift::gem_name.should == 'datashift'
+  end
+  
+  it "should provide root_path" do
+    DataShift.root_path.should_not be_empty
+  end
+  
+end