datashift 0.6.1 → 0.7.0

data/README.markdown

@@ -32,7 +32,7 @@ Many example Spreadsheets/CSV files in spec/fixtures, fully documented with comm
 
 Add gem 'datashift' to your Gemfile/bundle, or install the latest gem as usual :
 
-    `gem install datashift`
+    gem install datashift
 
 To use :
 
@@ -53,8 +53,8 @@ To use the Thor command line applications :
 
     Create a high level .thor file - e.g mysite.thor - in your applications root directory 
 
-Pull in the thor commands :
-```
+Edit the file and add the following to pull in the thor commands :
+```ruby
     require 'thor'
     require 'datashift'
 
@@ -68,7 +68,7 @@ and/or
 
     bundle exc thor list datashift
 
-To get usgae information use thor help <command>, for example
+To get usage information use thor help <command>, for example
 
     bundle exec thor help datashift:generate:excel
 

data/VERSION

@@ -1 +1 @@
-0.6.1
+0.7.0

data/datashift.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "datashift"
-  s.version = "0.6.1"
+  s.version = "0.7.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Thomas Statter"]
-  s.date = "2012-05-24"
+  s.date = "2012-05-27"
   s.description = "A suite of tools to move data between ActiveRecord models,databases,applications like Excel/Open Office, files and projects including Spree"
   s.email = "rubygems@autotelik.co.uk"
   s.extra_rdoc_files = [
@@ -24,6 +24,8 @@ Gem::Specification.new do |s|
     "README.rdoc",
     "Rakefile",
     "VERSION",
+    "datashift-0.6.0.gem",
+    "datashift-0.6.1.gem",
     "datashift.gemspec",
     "lib/applications/jruby/jexcel_file.rb",
     "lib/applications/jruby/word.rb",

data/lib/loaders/csv_loader.rb

@@ -32,10 +32,8 @@ module DataShift
       # For example if model has an attribute 'price' will map columns called Price, price, PRICE etc to this attribute
       map_headers_to_operators( @parsed_file.shift, options)
 
-      #if(options[:verbose])
       puts "\n\n\nLoading from CSV file: #{file_name}"
       puts "Processing #{@parsed_file.size} rows"
-      # end
 
       load_object_class.transaction do
         @loaded_objects =  []
@@ -66,7 +64,7 @@ module DataShift
           # TODO - handle when it's not valid ?
           # Process rest and dump out an exception list of Products ??
 
-          logger.info "Saving csv row #{row} to table object : #{load_object.inspect}" #if options[:verbose]
+          logger.info "Saving csv row #{row} to table object : #{load_object.inspect}"
 
           save
 

data/lib/loaders/loader_base.rb

@@ -28,7 +28,7 @@ module DataShift
 
     attr_accessor :loaded_objects, :failed_objects
 
-    attr_accessor :options
+    attr_accessor :options, :verbose
 
     # Support multiple associations being added to a base object to be specified in a single column.
     # 
@@ -100,7 +100,7 @@ module DataShift
       
       @method_mapper = DataShift::MethodMapper.new
       @options = options.clone
-      @verbose_logging = @options[:verbose_logging]
+      @verbose = @options[:verbose]
       @headers = []
 
       @default_data_objects ||= {}
@@ -214,21 +214,50 @@ module DataShift
       end
     end
     
-    
-    # Default values can be provided in YAML config file
+    def get_record_by(klazz, field, value)
+      x =  (@options[:sku_prefix]) ? "#{@options[:sku_prefix]}#{value}" : value
+
+      begin
+        if(@options[:case_sensitive]) 
+          puts "Search case sensitive for [#{x}] on #{field}" if(@verbose)
+          return klazz.send("find_by_#{field}", x)
+        elsif(@options[:use_like])
+          puts "Term : #{klazz}.where(\"#{field} LIKE '#{x}%'\")" if(@verbose)
+          return klazz.where("#{field} like ?", "#{x}%").first
+        else
+          puts "Term : #{klazz}.where(\"lower(#{field}) = '#{x.downcase}'\")" if(@verbose)
+          return klazz.where("lower(#{field}) = ?", x.downcase).first
+        end
+      rescue => e
+        logger.error("Exception attempting to find a record for [#{x}] on #{klazz}.#{field}")
+        logger.error e.backtrace
+        logger.error e.inspect
+        return nil
+      end
+    end
+      
+    # Default values and over rides can be provided in YAML config file.
+    # 
+    # Any Configuration under key 'LoaderBase' is merged into this classes
+    # existing options - taking precedence.
+    # 
     #  Format :
     #  
     #    LoaderClass:
     #     option: value
     #     
     #    Load Class:    (e.g Spree:Product)
-    #         atttribute: value
-    
+    #     datashift_defaults:     
+    #       value_as_string: "Default Project Value"  
+    #       category: reference:category_002    
+    #     
+    #     datashift_overrides:    
+    #       value_as_double: 99.23546
+    #
     def configure_from( yaml_file )
 
       data = YAML::load( File.open(yaml_file) )
       
-      
       # TODO - MOVE DEFAULTS TO OWN MODULE 
       # decorate the loading class with the defaults/ove rides to manage itself
       #   IDEAS .....
@@ -389,7 +418,7 @@ module DataShift
     end
 
     def save
-      #puts "DEBUG: SAVING #{load_object.class} : #{load_object.inspect}" #if(options[:verbose])
+      puts "DEBUG: SAVING #{load_object.class} : #{load_object.inspect}" if(@verbose)
       begin
         result = @load_object.save
         

data/lib/loaders/spreadsheet_loader.rb

@@ -39,8 +39,7 @@ module DataShift
 
         @excel = Spreadsheet.open file_name
         
-        #if(options[:verbose])
-        puts "\n\n\nLoading from Excel file: #{file_name}"
+        puts "\nLoading from Excel file: #{file_name}"
 
         sheet_number = options[:sheet_number] || 0
 

data/lib/loaders/spree/image_loader.rb

@@ -60,46 +60,46 @@ module DataShift
   end
 
   module SpreeHelper
-       
+     
+    # TODO - extract this out of SpreeHelper to create  a general paperclip loader
     class ImageLoader < LoaderBase
 
       include DataShift::ImageLoading
       include DataShift::CsvLoading
       include DataShift::ExcelLoading
       
-      def initialize(image = nil)
-        super( SpreeHelper::get_spree_class('Image'), image )
+      def initialize(image = nil, options = {})
+        super( SpreeHelper::get_spree_class('Image'), image, options )
+        
+        if(SpreeHelper::version.to_f > 1.0 )
+          @attachment_klazz  = DataShift::SpreeHelper::get_spree_class('Variant' )
+        else
+          @attachment_klazz  = DataShift::SpreeHelper::get_spree_class('Product' )
+        end
         
+        puts "Attachment Class is #{@attachment_klazz}" if(@verbose)
+          
         raise "Failed to create Image for loading" unless @load_object
       end
-
-      def sku_klazz
-        @sku_klazz ||= SpreeHelper::get_spree_class('Variant' )
-        @sku_klazz
-      end
       
       def process()
 
         if(current_value && @current_method_detail.operator?('attachment') )
+          
+          # assign the image file data as an attachment
           @load_object.attachment = get_file(current_value)
           
-          puts "Image attachment set : #{@load_object.inspect}"
+          puts "Image attachment created : #{@load_object.inspect}"
               
-        elsif(current_value && @current_method_detail.operator?('sku') )    
+        elsif(current_value && @current_method_detail.operator )    
           
-          return if(current_value.empty?)
-          puts "Looking for record with SKU #{current_value}"  
-          add_record( sku_klazz.find_by_sku(current_value) )
-               
-        elsif(current_value && @current_method_detail.operator?('name') )   
-                     
-          puts "Looking for record with NAME [#{current_value}]"
-          add_record attachment_klazz.find_by_name(current_value)
-        
+          # find the db record to assign our Image to
+          add_record( get_record_by(@attachment_klazz, @current_method_detail.operator, current_value) )
+                
         end
           
       end
-      
+    
       def add_record(record)
         if(record)
           if(SpreeHelper::version.to_f > 1 )
@@ -111,6 +111,7 @@ module DataShift
           puts "Image viewable set : #{record.inspect}"
           
         else
+          puts "WARNING - Cannot set viewable - No matching record supplied"
           logger.error"Failed to find a matching record"
         end
       end

data/lib/thor/spree/products_images.thor

@@ -55,7 +55,28 @@ module Datashift
 
       loader.perform_load(input, :mandatory => ['sku', 'name', 'price'] )
     end
+  
+
+    desc "attach_images", "Populate Products with images from Excel/CSV\nProvide column SKU or Name\nProvide column 'attachment' containing full path to image"
+    # :dummy => dummy run without actual saving to DB
+    method_option :input, :aliases => '-i', :required => true, :desc => "The 2 column import file (.xls or .csv)"
     
+    def attach_images()
+
+      require File.expand_path('config/environment.rb')
+      
+      require 'image_loader'
+      
+      image_klazz = DataShift::SpreeHelper::get_spree_class('Image' )
+      
+      # force inclusion means add to operator list even if not present
+      options = { :force_inclusion => ['sku', 'attachment'] } if(SpreeHelper::version.to_f > 1 )
+    
+      loader = DataShift::SpreeHelper::ImageLoader.new(nil, options)
+    
+      loader.perform_load( options[:input], options )
+    end
+  
     
     #
     # => rake datashift:spree:images input=vendor/extensions/site/fixtures/images
@@ -63,33 +84,37 @@ module Datashift
     #
     # => rake datashift:spree:images input=C:\images\taxon_icons skip_if_no_assoc=true klass=Taxon
     #
-    desc "images", "Populate the DB with images.\nDefault location db/image_seeds, or specify :input=<path> or dir under db/image_seeds with :folder"
+    desc "images", "Populate the DB with images from a directory where image names map to Product Sku/Name"
     
     # :dummy => dummy run without actual saving to DB
     method_option :input, :aliases => '-i', :required => true, :desc => "The import file (.xls or .csv)"
     
     method_option :process_when_no_assoc, :aliases => '-f', :type => :boolean, :desc => "Process image even if no Product found - force loading"
     
-
     method_option :sku, :aliases => '-s', :desc => "Lookup Product based on image name starting with sku"
     method_option :sku_prefix, :aliases => '-p', :desc => "Prefix to add to each SKU in import file"
     method_option :dummy, :aliases => '-d', :type => :boolean, :desc => "Dummy run, do not actually save Image or Product"
     method_option :verbose, :aliases => '-v', :type => :boolean, :desc => "Verbose logging"
     method_option :config, :aliases => '-c',  :type => :string, :desc => "Configuration file for Image Loader in YAML"
-    
-   
+    method_option :split_file_name_on,  :type => :string, :desc => "delimiter to progressivley split filename for Prod lookup", :default => '_'
+    method_option :case_sensitive, :type => :boolean, :desc => "Use case sensitive where clause to find Product"
+    method_option :use_like, :type => :boolean, :desc => "Use LIKE 'string%' instead of = 'string' in where clauses"
+  
     def images()
 
       require File.expand_path('config/environment.rb')
       
       require 'image_loader'
+            
+      @verbose = options[:verbose]
        
       puts "Using Product Name for lookup" unless(options[:sku])
       puts "Using SKU for lookup" if(options[:sku])
-           
+       
+      image_klazz = DataShift::SpreeHelper::get_spree_class('Image' )
+       
       attachment_klazz  = DataShift::SpreeHelper::get_spree_class('Product' )
       attachment_field  = 'name'
-      image_klazz       = DataShift::SpreeHelper::get_spree_class('Image' )
 
       if(options[:sku])
         attachment_klazz =  DataShift::SpreeHelper::get_spree_class('Variant' ) 
@@ -103,7 +128,7 @@ module Datashift
       #  raise "Could not find contant for model #{args[:model]}"
       #end
 
-      image_loader = DataShift::SpreeHelper::ImageLoader.new
+      image_loader = DataShift::SpreeHelper::ImageLoader.new(nil, options.dup)
 
       @loader_config = {}
       
@@ -121,84 +146,85 @@ module Datashift
       
       @image_cache = options[:input]
       
-      if(File.directory? @image_cache )
-        logger.info "Loading Spree images from #{@image_cache}"
+      unless(File.directory? @image_cache )
+        puts "ERROR: Supplied Path #{@image_cache} not accesible"
+        exit(-1)
+      end
+      
+      logger.info "Loading Spree images from #{@image_cache}"
 
-        missing_records = []
-        Dir.glob("#{@image_cache}/**/*.{jpg,png,gif}") do |image_name|
+      missing_records = []
+         
+      # unless record   # try splitting up filename in various ways looking for the SKU
+      split_on = @loader_config[:split_file_name_on] || options[:split_file_name_on]
+        
+      Dir.glob("#{@image_cache}/**/*.{jpg,png,gif}") do |image_name|
 
-          base_name = File.basename(image_name, '.*')
-          base_name.strip!
+        base_name = File.basename(image_name, '.*')
+        base_name.strip!
                        
-          logger.info "Processing image #{base_name} : #{File.exists?(image_name)}"
-           
-          record = nil
+        logger.info "Processing image file #{base_name} : #{File.exists?(image_name)}"
            
-          put "Processing image [#{base_name}]"
+        record = nil
            
-          # unless record   # try splitting up filename in various ways looking for the SKU
-          split_on = @loader_config[:split_file_name_on] || '_'
-              
-          base_name.split(split_on).each do |x| 
-            record = get_record_by(attachment_klazz, attachment_field, x)
-            break if record
-          end
+        puts "Search for product for image file [#{base_name}]" if(@verbose)
             
-          record = record.product if(record)  # SKU stored on Variant but we want it's master Product
+        record = image_loader.get_record_by(attachment_klazz, attachment_field, base_name)
+          
+        # try seperate portions of the filename, front -> back
+        base_name.split(split_on).each do |x| 
+          record = image_loader.get_record_by(attachment_klazz, attachment_field, x)
+          break if record
+        end unless(record)
+            
+        # this time try sequentially scanning
+        base_name.split(split_on).inject("") do |str, x| 
+          record = image_loader.get_record_by(attachment_klazz, attachment_field, "#{str}#{x}")
+          break if record
+          x
+        end unless(record)
+          
+        record = record.product if(record && record.respond_to?(:product))  # SKU stored on Variant but we want it's master Product
       
-          if(record)
-            logger.info "Found record for attachment : #{record.inspect}"
-            exists = record.images.detect {|i| puts "COMPARE #{i.attachment_file_name} => #{image_name}"; i.attachment_file_name == image_name }
+        if(record)
+          logger.info "Found record for attachment : #{record.inspect}"
+    
+          if(options[:skip_if_loaded])
+            exists = record.images.detect {|i| i.attachment_file_name == image_name }
             
-            if(options[:skip_if_loaded] && !exists.nil?)
-              logger.info "Skipping - Image #{image_name} already loaded for #{attachment_klazz}"
-              next
-            end
-          else
-            missing_records << image_name
+            logger.info "Skipping - Image #{image_name} already loaded for #{attachment_klazz}"
+            next if(exists)
           end
+        else
+          missing_records << image_name
+        end
           
-          next if(options[:dummy]) # Don't actually create/upload to DB if we are doing dummy run
-
-          # Check if Image must have an associated record
-          if(record || (record.nil? && options[:process_when_no_assoc]))
-            image_loader.reset()
-            puts "Processing Image #{image_name}"
-            image_loader.create_image(image_klazz, image_name, record)
-          end
-
+        next if(options[:dummy]) # Don't actually create/upload to DB if we are doing dummy run
+
+        # Check if Image must have an associated record
+        if(record || (record.nil? && options[:process_when_no_assoc]))
+          image_loader.reset()
+          puts "Adding Image #{image_name} to Product #{record.name}" if(@verbose)
+          logger.info("Adding Image #{image_name} to Product #{record.name}")
+          image_loader.create_image(image_klazz, image_name, record)
         end
 
-        unless missing_records.empty?
-          FileUtils.mkdir_p('MissingRecords') unless File.directory?('MissingRecords')
+      end
+
+      unless missing_records.empty?
+        FileUtils.mkdir_p('MissingRecords') unless File.directory?('MissingRecords')
         
-          puts '\nMISSING Records Report>>'
-          missing_records.each do |i|
-            puts "Copy #{i} to MissingRecords folder"
-            FileUtils.cp( i, 'MissingRecords')  unless(options[:dummy] == 'true')
-          end
+        puts 'MISSING Records Report>>'
+        missing_records.each do |i|
+          puts "Copy #{i} to MissingRecords folder"
+          FileUtils.cp( i, 'MissingRecords')  unless(options[:dummy] == 'true')
         end
-
-        puts "Dummy Run - if happy run without -d" if(options[:dummy])
-      else
-        puts "ERROR: Supplied Path #{@image_cache} not accesible"
-        exit(-1)
       end
-    end
-   
-    private
-    
-    def get_record_by(klazz, field, value)
-      x =  (options[:sku_prefix]) ? "#{options[:sku_prefix]}#{value}" : value
 
-      if(@loader_config['case_sensitive'])
-        puts "Search case sensitive for [#{x}] on #{field}"
-         return klazz.find(:first, :conditions => [ "? = ?", field, x ])
-      else
-        puts "Search for [#{x}] on #{field}"
-        return klazz.find(:first, :conditions => [ "lower(?) = ?", field, x.downcase ])
-      end
+      puts "Dummy Run Complete- if happy run without -d" if(options[:dummy])
+   
     end
+   
   end
 
 end

data/spec/spree_images_loader_spec.rb

@@ -101,9 +101,10 @@ describe 'SpreeImageLoading' do
     
     @Image_klass.all.size.should == 0
 
-    options = { :force_inclusion => ['sku', 'attachment'] } if(SpreeHelper::version.to_f > 1 )
+    # force inclusion means add to operator list even if not present
+    options = { :verbose => true, :force_inclusion => ['sku', 'attachment'] } if(SpreeHelper::version.to_f > 1 )
     
-    loader = DataShift::SpreeHelper::ImageLoader.new
+    loader = DataShift::SpreeHelper::ImageLoader.new(nil, options)
     
     loader.perform_load( SpecHelper::spree_fixture('SpreeImages.xls'), options )
     

metadata

@@ -2,7 +2,7 @@
 name: datashift
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.6.1
+  version: 0.7.0
 platform: ruby
 authors: 
   - Thomas Statter
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-05-24 00:00:00 Z
+date: 2012-05-27 00:00:00 Z
 dependencies: 
   - !ruby/object:Gem::Dependency 
     name: spreadsheet
@@ -40,6 +40,8 @@ files:
   - README.rdoc
   - Rakefile
   - VERSION
+  - datashift-0.6.0.gem
+  - datashift-0.6.1.gem
   - datashift.gemspec
   - lib/applications/jruby/jexcel_file.rb
   - lib/applications/jruby/word.rb