datashift 0.2.2 → 0.4.0

data/lib/loaders/spree/image_loader.rb

@@ -7,62 +7,64 @@ require 'loader_base'
 
 module DataShift
 
-   module ImageLoading
+
+  module ImageLoading
  
+    include DataShift::Logging
+     
     # Note the Spree Image model sets default storage path to
     # => :path => ":rails_root/public/assets/products/:id/:style/:basename.:extension"
 
-    def create_image(image_path, viewable_record = nil, options = {})
-
-      @@image_klass ||= SpreeHelper::get_spree_class('Image')
-      
-      image = @@image_klass.new
-      
-      unless File.exists?(image_path)
-        puts "ERROR : Invalid Path"
-        return image
+    def create_image(klass, image_path, viewable_record = nil, options = {})
+       
+      unless File.exists?(image_path) && File.readable?(image_path)
+        logger.error("Cannot process Image : Invalid Path #{image_path}")
+        raise "Cannot process Image : Invalid Path #{image_path}"
       end
-
+      
       alt = if(options[:alt])
         options[:alt]
       else
         (viewable_record and viewable_record.respond_to? :name) ? viewable_record.name : ""
       end
-      
-      image.alt = alt
-
-      begin
-        image.attachment = File.new(image_path, "r")
+     
+      file = begin
+        File.new(image_path, "rb")
       rescue => e
         puts e.inspect
-        puts "ERROR : Failed to read image #{image_path}"
-        return image
+        raise "ERROR : Failed to read image #{image_path}"
       end
 
-      image.attachment.reprocess!
-      image.viewable =  viewable_record if viewable_record
+      position = (viewable_record and viewable_record.respond_to?(:images)) ? viewable_record.images.length : 0
+      
+      image = klass.new( :attachment => file,:viewable => viewable_record, :alt => alt, :position => position)  
+      #image.attachment.reprocess!
+   
+      #image.viewable =  viewable_record if viewable_record
 
       puts image.save ? "Success: Created Image: #{image.inspect}" : "ERROR : Problem saving to DB Image: #{image.inspect}"
     end
   end
-  
-  class ImageLoader < LoaderBase
 
-    include DataShift::ImageLoading
+  module SpreeHelper
+       
+    class ImageLoader < LoaderBase
+
+      include DataShift::ImageLoading
+        
+      def initialize(image = nil)
+        puts SpreeHelper::get_spree_class('Image')
         
-    def initialize(image = nil)
-      @@image_klass ||= SpreeHelper::get_spree_class('Image')
+        @@image_klass ||= SpreeHelper::get_spree_class('Image')
             
-      super( @@image_klass, image )
-      raise "Failed to create Image for loading" unless @load_object
-    end
-
-    # Note the Spree Image model sets default storage path to
-    # => :path => ":rails_root/public/assets/products/:id/:style/:basename.:extension"
+        super( @@image_klass, image )
+        raise "Failed to create Image for loading" unless @load_object
+      end
 
-    def process( image_path, record = nil)
-      @load_object = create_image(path, record)
+      # The path to the physical image on local disk
+      def process(image_path, record = nil)
+        @load_object = create_image(@@image_klass, image_path, record)
+      end
     end
   end
-
 end

data/lib/loaders/spree/product_loader.rb

@@ -25,6 +25,7 @@ module DataShift
       def initialize(product = nil)
         super( SpreeHelper::get_product_class(), product, :instance_methods => true  )
      
+        @@image_klass ||= SpreeHelper::get_spree_class('Image')
         @@option_type_klass ||= SpreeHelper::get_spree_class('OptionType')
         @@option_value_klass ||= SpreeHelper::get_spree_class('OptionValue')
         @@property_klass ||= SpreeHelper::get_spree_class('Property')
@@ -42,6 +43,8 @@ module DataShift
       #   CSV files
       def perform_load( file_name, options = {} )
 
+        raise DataShift::BadFile, "Cannot load #{file_name} file not found." unless(File.exists?(file_name))
+        
         ext = File.extname(file_name)
           
         if(ext == '.xls')
@@ -50,7 +53,7 @@ module DataShift
         elsif(ext == '.csv')
           perform_csv_load(file_name, options)
         else
-          raise DataShift::UnsupportedFileType, "#{ext} files not supported - Try CSV or OpenOffice/Excel .xls"
+          raise DataShift::UnsupportedFileType, "#{ext} files not supported - Try .csv or OpenOffice/Excel .xls"
         end
       end
       
@@ -95,7 +98,7 @@ module DataShift
   
           # Spree has some stock management stuff going on, so dont usually assign to column vut use
           # on_hand and on_hand=
-          if(@load_object.variants.size > 0 && current_value.include?(LoaderBase::multi_assoc_delim))
+          if(@load_object.variants.size > 0 && current_value.to_s.include?(LoaderBase::multi_assoc_delim))
 
             #puts "DEBUG: COUNT_ON_HAND PER VARIANT",current_value.is_a?(String),
           
@@ -118,6 +121,13 @@ module DataShift
       end
 
       private
+      
+      # Take current column data and split into each association
+      # Supported Syntax :
+      #  assoc_find_name:value | assoc2_find_name:value | etc
+      def get_each_assoc
+        current_value.to_s.split( LoaderBase::multi_assoc_delim )
+      end
 
       # Special case for OptionTypes as it's two stage process
       # First add the possible option_types to Product, then we are able
@@ -128,7 +138,7 @@ module DataShift
         # TODO smart column ordering to ensure always valid by time we get to associations
         save_if_new
 
-        option_types = current_value.split( LoaderBase::multi_assoc_delim )
+        option_types = get_each_assoc#current_value.split( LoaderBase::multi_assoc_delim )
 
         option_types.each do |ostr|
           oname, value_str = ostr.split(LoaderBase::name_value_delim)
@@ -182,13 +192,13 @@ module DataShift
         # TODO smart column ordering to ensure always valid by time we get to associations
         save_if_new
 
-        images = current_value.split(LoaderBase::multi_assoc_delim)
+        images = get_each_assoc#current_value.split(LoaderBase::multi_assoc_delim)
 
         images.each do |image|
           
           img_path, alt_text = image.split(LoaderBase::name_value_delim)
           
-          image = create_image(img_path, @load_object, :alt => alt_text)
+          image = create_image(@@image_klass, img_path, @load_object, :alt => alt_text)
         end
       
       end
@@ -202,7 +212,7 @@ module DataShift
         # TODO smart column ordering to ensure always valid by time we get to associations
         save_if_new
 
-        property_list = current_value.split(LoaderBase::multi_assoc_delim)
+        property_list = get_each_assoc#current_value.split(LoaderBase::multi_assoc_delim)
 
         property_list.each do |pstr|
           pname, pvalue = pstr.split(LoaderBase::name_value_delim)
@@ -244,7 +254,7 @@ module DataShift
         # TODO smart column ordering to ensure always valid by time we get to associations
         save_if_new
 
-        chain_list = current_value().split(LoaderBase::multi_assoc_delim)
+        chain_list = get_each_assoc#current_value().split(LoaderBase::multi_assoc_delim)
 
         chain_list.each do |chain|
           

data/lib/thor/generate_excel.thor

@@ -3,7 +3,6 @@
 # Date ::     Mar 2012
 # License::   MIT.
 #
-#
 # Usage::
 #
 #  To pull Datashift commands into your main application :
@@ -14,29 +13,37 @@
 #
 #  Cmd Line:
 #
-# => bundle exec thor datashift:generate:excel --model <active record class> --result <output_template.xls>
+# => bundle exec thor datashift:generate:excel -m <active record class> -r <output_template.xls> -a
 #
-
+require 'datashift'
+  
+# Note, not DataShift, case sensitive, create namespace for command line : datashift
 module Datashift
- 
+
+          
   class Generate < Thor     
-                                             
-    desc "excel --model <Class> --result <file.xls>", "generate a template from an active record model" 
+  
+    include DataShift::Logging
+      
+    desc "excel", "generate a template from an active record model (with optional associations)" 
     method_option :model, :aliases => '-m', :required => true, :desc => "The active record model to export"
     method_option :result, :aliases => '-r', :required => true, :desc => "Create template of model in supplied file"
+    method_option :assoc, :aliases => '-a', :type => :boolean, :desc => "Include all associations in the template"
+    method_option :exclude, :aliases => '-e',  :type => :array, :desc => "Use with -a : Exclude association types. Any from #{DataShift::MethodDetail::supported_types_enum.to_a.inspect}"
     
     def excel()
      
-     # TODO - We're assuming run from a rails app/top level dir...
-     # ...can we make this more robust ? e.g what about when using active record but not in Rails app, 
-     require File.expand_path('config/environment.rb')
-
-      
-     require 'excel_generator'
+      # TODO - We're assuming run from a rails app/top level dir...
+      # ...can we make this more robust ? e.g what about when using active record but not in Rails app, 
+      require File.expand_path('config/environment.rb')
+   
+      require 'excel_generator'
 
       model = options[:model]
       result = options[:result]
-      
+     
+      logger.info "Datashift: Start Excel template generation in #{result}"
+            
       begin
         # support modules e.g "Spree::Property") 
         klass = ModelMapper::class_from_string(model)  #Kernel.const_get(model)
@@ -45,9 +52,22 @@ module Datashift
         raise "ERROR: No such Model [#{model}] found - check valid model supplied via -model <Class>"
       end
 
-      gen = DataShift::ExcelGenerator.new(result)
+      begin
+        gen = DataShift::ExcelGenerator.new(result)
+
+        if(options[:assoc])
+          opts = (options[:exclude]) ? {:exclude => options[:exclude]} : {}
+          logger.info("Datashift: Generating with associations")
+          gen.generate_with_associations(klass, opts)
+        else
+          gen.generate(klass)
+        end
+      rescue => e
+        puts e
+        puts e.backtrace
+        puts "Warning: Error during generation, template may be incomplete"
+      end
 
-      gen.generate(klass)
     end
   end
 

data/lib/thor/import_excel.thor

@@ -0,0 +1,84 @@
+# Copyright:: (c) Autotelik Media Ltd 2012
+# Author ::   Tom Statter
+# Date ::     Mar 2012
+# License::   MIT.
+#
+# Usage::
+#
+#  To pull Datashift commands into your main application :
+#
+#     require 'datashift'
+#
+#     DataShift::load_commands
+#
+#  Requires Jruby, cmd Line:
+#
+# => bundle exec thor datashift:import:excel -m <active record class> -r <output_template.xls> -a
+#
+#  Cmd Line:
+#
+# => jruby -S rake datashift:import:excel model=<active record class> input=<file.xls>
+# => jruby -S rake datashift:import:excel model=<active record class> input=C:\MyProducts.xlsverbose=true
+#
+require 'datashift'
+  
+
+# Note, not DataShift, case sensitive, create namespace for command line : datashift
+module Datashift
+
+          
+  class Import < Thor     
+  
+    include DataShift::Logging
+      
+    desc "excel", "import .xls file for specifiec active record model" 
+    method_option :model, :aliases => '-m', :required => true, :desc => "The related active record model"
+    method_option :input, :aliases => '-i', :required => true, :desc => "The input .xls file"
+    method_option :assoc, :aliases => '-a', :type => :boolean, :desc => "Include any associations supplied in the input"
+    method_option :exclude, :aliases => '-e',  :type => :array, :desc => "Use with -a : Exclude association types. Any from #{DataShift::MethodDetail::supported_types_enum.to_a.inspect}"
+    
+    def excel()
+     
+      # TODO - We're assuming run from a rails app/top level dir...
+      # ...can we make this more robust ? e.g what about when using active record but not in Rails app, 
+      require File.expand_path('config/environment.rb')
+   
+      require 'excel_loader'
+
+      model = options[:model]
+      begin
+        # support modules e.g "Spree::Property") 
+        klass = ModelMapper::class_from_string(model)  #Kernel.const_get(model)
+      rescue NameError
+        raise "ERROR: No such AR Model found - check valid model supplied via model=<Class>"
+      end
+
+      if(ENV['loader'])
+        begin
+          #loader_klass = Kernel.const_get(ENV['loader'])
+          # support modules e.g "Spree::Property") 
+          loader_klass = ModelMapper::class_from_string(ENV['loader'])  #Kernel.const_get(model)
+
+          loader = loader_klass.new(klass)
+
+          logger.info("INFO: Using loader : #{loader.class}")
+        rescue
+          logger.error("INFO: No specific #{model}Loader found  - using generic ExcelLoader")
+          loader = DataShift::ExcelLoader.new(klass)
+        end
+      else
+        logger.info("No Loader specified - using generic ExcelLoader")
+        loader = DataShift::ExcelLoader.new(klass)
+      end
+
+      logger.info("ARGS #{options.inspect} [#{options[:verbose]}]")
+      loader.logger.verbose if(ENV['verbose'])
+      
+      loader.configure_from( ENV['config'] ) if(ENV['config'])
+       
+      loader.perform_load(options[:input])
+    end
+  end
+
+end
+

data/lib/thor/spree/bootstrap_cleanup.thor

@@ -0,0 +1,33 @@
+# Copyright:: (c) Autotelik Media Ltd 2012
+# Author ::   Tom Statter
+# Date ::     March 2012
+# License::   MIT. Free, Open Source.
+#
+# Usage::
+# bundle exec thor help datashift:spreeboot
+# bundle exec thor datashift:spreeboot:cleanup
+#
+# Note, not DataShift, case sensitive, create namespace for command line : datashift
+module Datashift
+        
+  class Spreeboot < Thor     
+  
+    include DataShift::Logging
+       
+    desc "cleanup", "Remove Spree Product/Variant data from DB"
+    
+    def cleanup()  #, [:input, :verbose, :sku_prefix] => :environment do |t, args|
+
+     require 'spree_helper'
+
+      require File.expand_path('config/environment.rb')
+    
+      %w{Image OptionType OptionValue Product Property ProductProperty Variant Taxonomy Taxon Zone}.each do |k|
+        instance_variable_set("@#{k}_klass", DataShift::SpreeHelper::get_spree_class(k)) 
+        instance_variable_get("@#{k}_klass").delete_all
+      end
+    end
+  
+  end
+
+end

data/lib/thor/spree/products_images.thor

@@ -0,0 +1,171 @@
+# Copyright:: (c) Autotelik Media Ltd 2012
+# Author ::   Tom Statter
+# Date ::     March 2012
+# License::   MIT. Free, Open Source.
+#
+# Usage::
+# bundle exec thor help datashift:spree
+# bundle exec thor datashift:spree:products -i db/datashift/MegamanFozz20111115_load.xls -s 299S_
+#
+# bundle exec thor  datashift:spree:images -i db/datashift/imagebank -s -p 299S_
+#
+
+# Note, not DataShift, case sensitive, create namespace for command line : datashift
+module Datashift
+        
+  class Spree < Thor     
+  
+    include DataShift::Logging
+       
+    desc "products", "Populate Spree Product/Variant data from .xls (Excel) or CSV file"
+     
+    method_option :input, :aliases => '-i', :required => true, :desc => "The import file (.xls or .csv)"
+    method_option :sku_prefix, :aliases => '-s', :desc => "Prefix to add to each SKU in import file"
+    method_option :verbose, :aliases => '-v', :type => :boolean, :desc => "Verbose logging"
+    method_option :config, :aliases => '-c',  :type => :string, :desc => "Configuration file containg defaults or over rides in YAML"
+    
+    def products()  #, [:input, :verbose, :sku_prefix] => :environment do |t, args|
+
+      # TODO - We're assuming run from a rails app/top level dir...
+      # ...can we make this more robust ? e.g what about when using active record but not in Rails app, 
+      require File.expand_path('config/environment.rb')
+      
+      input = options[:input]
+
+      require 'product_loader'
+
+      loader = DataShift::SpreeHelper::ProductLoader.new
+
+      # YAML configuration file to drive defaults etc
+
+      if(options[:config])
+        raise "Bad Config - Cannot find specified file #{options[:config]}" unless File.exists?(options[:config])
+        
+        loader.configure_from( options[:config] )
+      else
+        loader.set_default_value('available_on', Time.now.to_s(:db) )
+        loader.set_default_value('cost_price', 0.0 )
+      end
+      
+      loader.set_prefix('sku', options[:sku_prefix] ) if(options[:sku_prefix])
+      
+      puts "DataShift::Product starting upload from file: #{input}"
+
+      loader.perform_load(input, :mandatory => ['sku', 'name', 'price'] )
+    end
+    
+    
+    #
+    # => rake datashift:spree:images input=vendor/extensions/site/fixtures/images
+    # => rake datashift:spree:images input=C:\images\photos large dummy=true
+    #
+    # => rake datashift:spree:images input=C:\images\taxon_icons skip_if_no_assoc=true klass=Taxon
+    #
+    desc "images", "Populate the DB with images.\nDefault location db/image_seeds, or specify :input=<path> or dir under db/image_seeds with :folder"
+    
+    # :dummy => dummy run without actual saving to DB
+    method_option :input, :aliases => '-i', :required => true, :desc => "The import file (.xls or .csv)"
+    
+    method_option :process_when_no_assoc, :aliases => '-f', :type => :boolean, :desc => "Process image even if no Product found - force loading"
+    
+
+    method_option :sku, :aliases => '-s', :desc => "Lookup Product based on image name starting with sku"
+    method_option :sku_prefix, :aliases => '-p', :desc => "Prefix to add to each SKU in import file"
+    method_option :dummy, :aliases => '-d', :type => :boolean, :desc => "Dummy run, do not actually save Image or Product"
+    method_option :verbose, :aliases => '-v', :type => :boolean, :desc => "Verbose logging"
+    #method_option :config, :aliases => '-c',  :type => :string, :desc => "Configuration file containg defaults or over rides in YAML"
+   
+    def images()#, [:input, :folder, :dummy, :sku, :skip_if_no_assoc, :skip_if_loaded, :model] => :environment do |t, args|
+
+      require File.expand_path('config/environment.rb')
+      
+      require 'image_loader'
+
+      @image_cache = options[:input]
+        
+      attachment_klazz  = DataShift::SpreeHelper::get_spree_class('Product' )
+      sku_klazz         = DataShift::SpreeHelper::get_spree_class('Variant' )
+
+      # TODO generalise for any paperclip project, for now just Spree
+      #begin
+      #  attachment_klazz = Kernel.const_get(args[:model]) if(args[:model])
+      # rescue NameError
+      #  raise "Could not find contant for model #{args[:model]}"
+      #end
+
+      image_loader = DataShift::SpreeHelper::ImageLoader.new
+
+      if(File.directory? @image_cache )
+        logger.info "Loading Spree images from #{@image_cache}"
+
+        missing_records = []
+        Dir.glob("#{@image_cache}/**/*.{jpg,png,gif}") do |image_name|
+
+          base_name = File.basename(image_name, '.*')
+           
+          logger.info "Processing #{base_name} : #{File.exists?(image_name)}"
+           
+          record = nil
+          if(options[:sku])
+            sku = base_name.slice!(/\w+/)
+            sku.strip!
+            base_name.strip!
+            
+            sku = "#{options[:sku_prefix]}#{sku}" if(options[:sku_prefix])
+
+            record = sku_klazz.find_by_sku(sku)
+            
+            unless record   # try splitting up filename in various ways looking for the SKU
+              sku.split( '_' ).each do |x| 
+                x = "#{options[:sku_prefix]}#{x}" if(options[:sku_prefix])
+                record = sku_klazz.find_by_sku(x)
+                break if record
+              end
+            end
+            
+            record = record.product if(record)  # SKU stored on Variant but we want it's master Product
+            
+          else
+            record = attachment_klazz.find_by_name(base_name)
+          end
+      
+          if(record)
+            logger.info "Found record for attachment : #{record.inspect}"
+            exists = record.images.detect {|i| puts "COMPARE #{i.attachment_file_name} => #{image_name}"; i.attachment_file_name == image_name }
+            
+            if(options[:skip_if_loaded] && !exists.nil?)
+              logger.info "Skipping - Image #{image_name} already loaded for #{attachment_klazz}"
+              next
+            end
+          else
+            missing_records << image_name
+          end
+
+          # Now do actual upload to DB unless we are doing a dummy run,
+          # or the Image must have an associated record
+          unless(options[:dummy] == 'true' || (options[:process_when_no_assoc] && record.nil?))
+            image_loader.reset()
+            puts "Process Image"
+            image_loader.process( image_name, record )
+          end
+
+        end
+
+        unless missing_records.empty?
+          FileUtils.mkdir_p('MissingRecords') unless File.directory?('MissingRecords')
+        
+          puts '\nMISSING Records Report>>'
+          missing_records.each do |i|
+            puts "Copy #{i} to MissingRecords folder"
+            FileUtils.cp( i, 'MissingRecords')  unless(options[:dummy] == 'true')
+          end
+        end
+
+      else
+        puts "ERROR: Supplied Path #{@image_cache} not accesible"
+        exit(-1)
+      end
+    end
+  end
+
+end