datashift 0.7.0 → 0.8.0

data/lib/loaders/spree/product_loader.rb

@@ -39,6 +39,12 @@ module DataShift
         logger.debug "PRODUCT #{@load_object.inspect} MASTER: #{@load_object.master.inspect}"
       end
 
+      def perform_load( file_name, options = {} )
+         # In >= 1.1.0 Image moved to master Variant from Product so no association called Images on Product anymore
+        options[:force_inclusion] = options[:force_inclusion] ? ['images'] : [*options[:force_inclusion]] << 'images'
+    
+        super(file_name, options)
+      end
 
       # Over ride base class process with some Spree::Product specifics
       #
@@ -120,68 +126,100 @@ module DataShift
       # Special case for OptionTypes as it's two stage process
       # First add the possible option_types to Product, then we are able
       # to define Variants on those options values.
+      #  To defiene a Variant : 
+      #  1) define at least one OptionType on Product, for example Size
+      #  2) Provide a value for at least one of these OptionType
+      #  3) A composite Variant can be created by supplyiung a vlaue for more than one OptionType
+      #       fro example Colour : Red and Size Medium
+      # Supported Syntax :
+      #  '|' seperates Variants
+      #   ',' list of option values
+      #  Examples : 
+      #     mime_type:jpeg;print_type:black_white|mime_type:jpeg|mime_type:png, PDF;print_type:colour
       #
       def add_options
         
         # TODO smart column ordering to ensure always valid by time we get to associations
         save_if_new
 
-        option_types = get_each_assoc#current_value.split( LoaderBase::multi_assoc_delim )
-
-        option_types.each do |ostr|
-          oname, value_str = ostr.split(LoaderBase::name_value_delim)
+        # example : mime_type:jpeg;print_type:black_white|mime_type:jpeg|mime_type:png, PDF;print_type:colour
+        
+        variants = get_each_assoc#current_value.split( LoaderBase::multi_assoc_delim )
 
-          option_type = @@option_type_klass.find_by_name(oname)
+        # 1) mime_type:jpeg;print_type:black_white  2) mime_type:jpeg  3) mime_type:png, PDF;print_type:colour
 
-          unless option_type
-            option_type = @@option_type_klass.create( :name => oname, :presentation => oname.humanize)
-            # TODO - dynamic creation should be an option
+        variants.each do |per_variant| 
+                
+          option_types = per_variant.split(';')    # [mime_type:jpeg, print_type:black_white]   
+          
+          optiontype_vlist_map = {}
+                      
+          option_types.each do |ostr|
+             
+            oname, value_str = ostr.split(LoaderBase::name_value_delim)
+    
+            option_type = @@option_type_klass.find_by_name(oname)
 
             unless option_type
-              puts "WARNING: OptionType #{oname} NOT found - Not set Product"
-              next
+              option_type = @@option_type_klass.create( :name => oname, :presentation => oname.humanize)
+              # TODO - dynamic creation should be an option
+
+              unless option_type
+                puts "WARNING: OptionType #{oname} NOT found and could not create - Not set Product"
+                next
+              end
+              puts "Created missing OptionType #{option_type.inspect}"
             end
-            puts "Created missing OptionType #{option_type.inspect}"
-          end
 
-          @load_object.option_types << option_type unless @load_object.option_types.include?(option_type)
+            # OptionTypes must be specified first on Product to enable Variants to be created
+            # TODO - is include? very inefficient ??
+            @load_object.option_types << option_type unless @load_object.option_types.include?(option_type)
 
-          # Can be simply list of OptionTypes, some or all without values
-          next unless(value_str)
+            # Can be simply list of OptionTypes, some or all without values
+            next unless(value_str)
 
-          # Now get the value(s) for the option e.g red,blue,green for OptType 'colour'
-          ovalues = value_str.split(',')
+            optiontype_vlist_map[option_type] = []
+            
+            # Now get the value(s) for the option e.g red,blue,green for OptType 'colour'   
+            optiontype_vlist_map[option_type] = value_str.split(',')
+          end
+         
+          next if(optiontype_vlist_map.empty?) # only option types specified - no values
 
+          # Now create set of Variants, some of which maybe composites           
+          # Find the longest set of OVs to use as base for combining with the rest
+          sorted_map = optiontype_vlist_map.sort_by { |k,v| v.size }.reverse
+          
+          # ovalues = 'pdf','jpeg','png'
+          option_type, ovalues = sorted_map.shift
           # TODO .. benchmarking to find most efficient way to create these but ensure Product.variants list
           # populated .. currently need to call reload to ensure this (seems reqd for Spree 1/Rails 3, wasn't required b4
-          ovalues.each_with_index do |ovname, i|
-            ovname.strip!
-            ov = @@option_value_klass.find_or_create_by_name_and_option_type_id(ovname, option_type.id)
-            if ov
-              begin
-                # This one line seems to works for 1.1.0 - 3.2 but not 1.0.0 - 3.1 ??
-                if(SpreeHelper::version.to_f >= 1.1)
-                  variant = @load_object.variants.create( :sku => "#{@load_object.sku}_#{i}", :price => @load_object.price)
-                else
-                  variant = @@variant_klass.create( :product => @load_object, :sku => "#{@load_object.sku}_#{i}", :price => @load_object.price, :available_on => @load_object.available_on)
-                  #if(variant.valid?)
-                  # variant.save
-                  # @load_object.variants << variant
-               
-                  #else
-                  #puts "WARNING: For Option #{ovname} - Variant creation failed #{variant.errors.inspect}"
-                end
-                variant.option_values << ov
-              rescue =>  e
-                puts "Failed to create a Variant for Product #{@load_object.name}"
-                puts e.inspect
-                #puts e.backtrace
-              end
+          ovalues.each do |ovname|
               
-              logger.debug "Created New Variant: #{variant.inspect}"
+            ov_list = []
+          
+            ov = @@option_value_klass.find_or_create_by_name_and_option_type_id(ovname.strip, option_type.id)
+                
+            ov_list << ov if ov
               
-            else
-              puts "WARNING: Option #{ovname} NOT FOUND - No Variant created"
+            sorted_map.each do |ot, ovlist| ovlist.each do |for_composite|
+                ov = @@option_value_klass.find_or_create_by_name_and_option_type_id(for_composite.strip, ot.id)
+                
+                ov_list << ov if(ov)
+              end 
+            end
+
+            unless(ov_list.empty?)
+              i = @load_object.variants.size + 1
+              
+              # This one line seems to works for 1.1.0 - 3.2 but not 1.0.0 - 3.1 ??
+              if(SpreeHelper::version.to_f >= 1.1)
+                variant = @load_object.variants.create( :sku => "#{@load_object.sku}_#{i}", :price => @load_object.price)
+              else
+                variant = @@variant_klass.create( :product => @load_object, :sku => "#{@load_object.sku}_#{i}", :price => @load_object.price, :available_on => @load_object.available_on)
+              end
+
+              variant.option_values << ov_list if(variant)
             end
           end
           
@@ -190,7 +228,7 @@ module DataShift
           #puts "DEBUG Load Object now has Variants : #{@load_object.variants.inspect}"
         end
 
-      end
+      end # each Variant
 
       # Special case for Images
       # A list of paths to Images with a optional 'alt' value - supplied in form :
@@ -203,10 +241,13 @@ module DataShift
         images = get_each_assoc#current_value.split(LoaderBase::multi_assoc_delim)
 
         images.each do |image|
-          
+          puts "Add Image #{image}"
           img_path, alt_text = image.split(LoaderBase::name_value_delim)
           
-          image = create_image(@@image_klass, img_path, @load_object, :alt => alt_text)
+          # moved from Prod to Variant in spree 1.x.x
+          attachment  = (SpreeHelper::version.to_f > 1) ? @load_object.master : @load_object
+          
+          image = create_image(@@image_klass, img_path, attachment, :alt => alt_text)
           logger.debug("Product assigned an Image : #{image.inspect}")
         end
       
@@ -226,12 +267,13 @@ module DataShift
         property_list = get_each_assoc#current_value.split(LoaderBase::multi_assoc_delim)
 
         property_list.each do |pstr|
-          
-        find_by_name, find_by_value = get_find_operator_and_rest( pstr )
-                 
-        raise "Cannot find Property via #{find_by_name} (with value #{find_by_value})" unless(find_by_name)
+        
+          # Special case, we know we lookup on name so operator is effectively the name to lookup
+          find_by_name, find_by_value = get_find_operator_and_rest( pstr )
+     
+          raise "Cannot find Property via #{find_by_name} (with value #{find_by_value})" unless(find_by_name)
                         
-        property = @@property_klass.find_by_name(find_by_name)
+          property = @@property_klass.find_by_name(find_by_name)
 
           unless property
             property = @@property_klass.create( :name => find_by_name, :presentation => find_by_name.humanize)
@@ -244,8 +286,8 @@ module DataShift
               x = @@product_property_klass.new( :value => find_by_value )
               x.property = property
               x.save
-              logger.info "Created New ProductProperty #{x.inspect}"
               @load_object.product_properties << x 
+              logger.info "Created New ProductProperty #{x.inspect}"
             else
               @load_object.product_properties << @@product_property_klass.create( :property => property, :value => find_by_values)
             end

data/lib/thor/generate_excel.thor

@@ -13,7 +13,8 @@
 #
 #  Cmd Line:
 #
-# => bundle exec thor datashift:generate:excel -m <active record class> -r <output_template.xls> -a
+# => bundle exec thor list datashift
+#    bundle exec thor help  datashift:generate:excel
 #
 require 'datashift'
   
@@ -49,9 +50,11 @@ module Datashift
         klass = ModelMapper::class_from_string(model)  #Kernel.const_get(model)
       rescue NameError => e
         puts e
-        raise "ERROR: No such Model [#{model}] found - check valid model supplied via -model <Class>"
+        raise Thor::Error.new("ERROR: No such Model [#{model}] found - check valid model supplied")
       end
 
+      raise Thor::Error.new("ERROR: No such Model [#{model}] found - check valid model supplied") unless(klass)
+      
       begin
         gen = DataShift::ExcelGenerator.new(result)
 

data/lib/thor/spree/bootstrap_cleanup.thor

@@ -8,6 +8,10 @@
 # bundle exec thor datashift:spreeboot:cleanup
 #
 # Note, not DataShift, case sensitive, create namespace for command line : datashift
+
+require File.expand_path('config/environment.rb')
+
+
 module Datashift
         
   class Spreeboot < Thor     
@@ -22,31 +26,33 @@ module Datashift
 
       require File.expand_path('config/environment.rb')
 
+      ActiveRecord::Base.connection.execute("TRUNCATE spree_products_taxons")
+      
       cleanup =  %w{ Image OptionType OptionValue 
                     Product Property ProductGroup ProductProperty ProductOptionType 
-                    Variant Taxonomy Taxon Zone
+                    Variant Taxonomy Taxon
       }
 
       cleanup.each do |k|
         klass = DataShift::SpreeHelper::get_spree_class(k)
-      if(klass)
-        puts "Clearing model #{klass}"
-        klass.delete_all
-      else
-        puts "WARNING - Coulod not find AR model for class name #{k}"
+        if(klass)
+          puts "Clearing model #{klass}"
+          klass.delete_all
+        else
+          puts "WARNING - Could not find AR model for class name #{k}"
+        end
       end
-        
-    end
       
-    if(File.exists?('public/spree/products') )
-      puts "Removing old Product assets from 'public/spree/products'"
-      FileUtils::rm_rf('public/public/spree/products') 
-    end
+      image_bank = 'public/spree/products'
       
-    FileUtils::rm_rf('MissingRecords') if(File.exists?('MissingRecords') )
+      if(File.exists?(image_bank) )
+        puts "Removing old Product assets from '#{image_bank}'"
+        FileUtils::rm_rf(image_bank) 
+      end
       
-  end
+      FileUtils::rm_rf('MissingRecords') if(File.exists?('MissingRecords') )
+      
+    end
   
-end
-
+  end
 end

data/lib/thor/spree/products_images.thor

@@ -21,7 +21,7 @@ module Datashift
     desc "products", "Populate Spree Product/Variant data from .xls (Excel) or CSV file"
      
     method_option :input, :aliases => '-i', :required => true, :desc => "The import file (.xls or .csv)"
-    method_option :sku_prefix, :aliases => '-s', :desc => "Prefix to add to each SKU in import file"
+    method_option :sku_prefix, :aliases => '-s', :desc => "Prefix to add to each SKU before saving Product"
     method_option :verbose, :aliases => '-v', :type => :boolean, :desc => "Verbose logging"
     method_option :config, :aliases => '-c',  :type => :string, :desc => "Configuration file containg defaults or over rides in YAML"
     
@@ -53,7 +53,12 @@ module Datashift
       
       puts "DataShift::Product starting upload from file: #{input}"
 
-      loader.perform_load(input, :mandatory => ['sku', 'name', 'price'] )
+      options = {:mandatory => ['sku', 'name', 'price']}
+    
+      # In >= 1.1.0 Image moved to master Variant from Product
+      options[:force_inclusion] = ['images'] if(DataShift::SpreeHelper::version.to_f > 1 )
+      
+      loader.perform_load(input, options)
     end
   
 
@@ -79,32 +84,36 @@ module Datashift
   
     
     #
-    # => rake datashift:spree:images input=vendor/extensions/site/fixtures/images
+    # => thor datashift:spree:images input=vendor/extensions/site/fixtures/images
     # => rake datashift:spree:images input=C:\images\photos large dummy=true
     #
     # => rake datashift:spree:images input=C:\images\taxon_icons skip_if_no_assoc=true klass=Taxon
     #
-    desc "images", "Populate the DB with images from a directory where image names map to Product Sku/Name"
+    desc "images", "Populate the DB with images from a directory\nwhere image names contain somewhere the Product Sku/Name"
     
     # :dummy => dummy run without actual saving to DB
-    method_option :input, :aliases => '-i', :required => true, :desc => "The import file (.xls or .csv)"
-    
-    method_option :process_when_no_assoc, :aliases => '-f', :type => :boolean, :desc => "Process image even if no Product found - force loading"
+    method_option :input, :aliases => '-i', :required => true, :desc => "The input path containing images (.jpg, .gif, .png)"
     
+    method_option :recursive, :aliases => '-r', :type => :boolean, :desc => "Scan sub directories of input for images"
+     
     method_option :sku, :aliases => '-s', :desc => "Lookup Product based on image name starting with sku"
-    method_option :sku_prefix, :aliases => '-p', :desc => "Prefix to add to each SKU in import file"
+    method_option :sku_prefix, :aliases => '-p', :desc => "Prefix to add to each SKU in import file before attempting lookup"
     method_option :dummy, :aliases => '-d', :type => :boolean, :desc => "Dummy run, do not actually save Image or Product"
+    
+    method_option :process_when_no_assoc, :aliases => '-f', :type => :boolean, :desc => "Process image even if no Product found - force loading"
+    method_option :skip_when_assoc, :aliases => '-x', :type => :boolean, :desc => "DO not process image if Product already has image"
+    
     method_option :verbose, :aliases => '-v', :type => :boolean, :desc => "Verbose logging"
     method_option :config, :aliases => '-c',  :type => :string, :desc => "Configuration file for Image Loader in YAML"
     method_option :split_file_name_on,  :type => :string, :desc => "delimiter to progressivley split filename for Prod lookup", :default => '_'
     method_option :case_sensitive, :type => :boolean, :desc => "Use case sensitive where clause to find Product"
-    method_option :use_like, :type => :boolean, :desc => "Use LIKE 'string%' instead of = 'string' in where clauses"
+    method_option :use_like, :type => :boolean, :desc => "Use sku/name LIKE 'string%' instead of sku/name = 'string' in where clauses to find Product"
   
     def images()
 
       require File.expand_path('config/environment.rb')
       
-      require 'image_loader'
+      require 'spree/image_loader'
             
       @verbose = options[:verbose]
        
@@ -116,59 +125,53 @@ module Datashift
       attachment_klazz  = DataShift::SpreeHelper::get_spree_class('Product' )
       attachment_field  = 'name'
 
-      if(options[:sku])
+      if(options[:sku] || SpreeHelper::version.to_f > 1)
         attachment_klazz =  DataShift::SpreeHelper::get_spree_class('Variant' ) 
         attachment_field = 'sku'
       end
-      
-      # TODO generalise for any paperclip project, for now just Spree
-      #begin
-      #  attachment_klazz = Kernel.const_get(args[:model]) if(args[:model])
-      # rescue NameError
-      #  raise "Could not find contant for model #{args[:model]}"
-      #end
 
-      image_loader = DataShift::SpreeHelper::ImageLoader.new(nil, options.dup)
-
-      @loader_config = {}
-      
+      image_loader = DataShift::SpreeHelper::ImageLoader.new(nil, options)
+ 
       if(options[:config])
         raise "Bad Config - Cannot find specified file #{options[:config]}" unless File.exists?(options[:config])
         
         image_loader.configure_from( options[:config] )
-        
-        @loader_config = YAML::load( File.open(options[:config]) )
-        
-        @loader_config = @loader_config['ImageLoader']
       end
+
+      loader_config = image_loader.options
+ 
+      puts "CONFIG: #{loader_config.inspect}"
+      puts "OPTIONS #{options.inspect}"
       
-      puts "CONFIG: #{@loader_config.inspect}"
-      
-      @image_cache = options[:input]
+      @image_path = options[:input]
       
-      unless(File.directory? @image_cache )
-        puts "ERROR: Supplied Path #{@image_cache} not accesible"
+      unless(File.exists?(@image_path))
+        puts "ERROR: Supplied Path [#{@image_path}] not accesible"
         exit(-1)
       end
       
-      logger.info "Loading Spree images from #{@image_cache}"
+      logger.info "Loading Spree images from #{@image_path}"
 
       missing_records = []
          
-      # unless record   # try splitting up filename in various ways looking for the SKU
-      split_on = @loader_config[:split_file_name_on] || options[:split_file_name_on]
-        
-      Dir.glob("#{@image_cache}/**/*.{jpg,png,gif}") do |image_name|
+      # try splitting up filename in various ways looking for the SKU
+      split_on = loader_config['split_file_name_on'] || options[:split_file_name_on]
+       
+      puts "Will scan image names splitting on delimiter : #{split_on}"
+      
+      image_cache = DataShift::ImageLoading::get_files(@image_path, options)
+      
+      image_cache.each do |image_name|
 
+        image_base_name = File.basename(image_name)
+        
         base_name = File.basename(image_name, '.*')
         base_name.strip!
                        
         logger.info "Processing image file #{base_name} : #{File.exists?(image_name)}"
            
         record = nil
-           
-        puts "Search for product for image file [#{base_name}]" if(@verbose)
-            
+                   
         record = image_loader.get_record_by(attachment_klazz, attachment_field, base_name)
           
         # try seperate portions of the filename, front -> back
@@ -189,11 +192,17 @@ module Datashift
         if(record)
           logger.info "Found record for attachment : #{record.inspect}"
     
-          if(options[:skip_if_loaded])
-            exists = record.images.detect {|i| i.attachment_file_name == image_name }
+          if(options[:skip_when_assoc])
             
-            logger.info "Skipping - Image #{image_name} already loaded for #{attachment_klazz}"
-            next if(exists)
+            paper_clip_name = image_base_name.gsub(Paperclip::Attachment::default_options[:restricted_characters], '_')
+            
+            exists = record.images.detect {|i| puts "Check #{paper_clip_name} matches #{i.attachment_file_name}"; i.attachment_file_name == paper_clip_name }
+            if(exists)
+              rid = record.respond_to?(:name) ? record.name : record.id
+              puts "Skipping Image #{image_name} already loaded for #{rid}"
+              logger.info "Skipping - Image #{image_name} already loaded for #{attachment_klazz}"
+              next 
+            end
           end
         else
           missing_records << image_name
@@ -204,9 +213,10 @@ module Datashift
         # Check if Image must have an associated record
         if(record || (record.nil? && options[:process_when_no_assoc]))
           image_loader.reset()
-          puts "Adding Image #{image_name} to Product #{record.name}" if(@verbose)
+          
           logger.info("Adding Image #{image_name} to Product #{record.name}")
           image_loader.create_image(image_klazz, image_name, record)
+          puts "Added Image #{File.basename(image_name)} to Product #{record.sku} : #{record.name}" if(@verbose)
         end
 
       end
@@ -214,11 +224,14 @@ module Datashift
       unless missing_records.empty?
         FileUtils.mkdir_p('MissingRecords') unless File.directory?('MissingRecords')
         
-        puts 'MISSING Records Report>>'
+        puts "WARNING : #{missing_records.size} of #{image_cache.size} images could not be attached to a Product"
+        puts 'Copying all images with MISSING Records to ./MissingRecords >>'
         missing_records.each do |i|
           puts "Copy #{i} to MissingRecords folder"
           FileUtils.cp( i, 'MissingRecords')  unless(options[:dummy] == 'true')
         end
+      else
+        puts "All images (#{image_cache.size}) were succesfully attached to a Product"
       end
 
       puts "Dummy Run Complete- if happy run without -d" if(options[:dummy])