datashift 0.6.0 → 0.6.1

data/README.markdown

@@ -2,29 +2,31 @@
 
 Provides tools to shift data between Ruby, external applications, files and ActiveRecord.
 
-Wiki taking shape with more info here : <b>https://github.com/autotelik/datashift/wiki</b>
+Specific loaders and command line tasks for Spree E-Commerce.
 
-### Features
+Wiki taking shape with more info here : **https://github.com/autotelik/datashift/wiki**
 
-Map Active Record models and associations to files, generate sample templates.
+### Features
 
 Import and Export ActiveRecord models through .xls or CSV  files, including
 all associations and with configurable defaults.
 
+Export all data or simply generate a sample template with headers only.
+
 Create, parse and use Excel/OpenOffice (.xls) documents dynamically from Ruby.
 
 Easily extendable Loader functionality to deal with non trivial import cases, such
 as complex association lookups.
 
-High level rake/thor command line tasks for import/export provided.
+High level rake and thor command line tasks for import/export provided.
 
-Specific loaders and rake tasks for Spree E-Commerce, enabling import/export of all data including Products with
-complex associations such as Properties/Taxons/Options/Variants and Images.
+Specific loaders and command line tasks provided out the box for **Spree E-Commerce**, 
+enabling import/export of Product data including creating Variants with different
+ count on hands and all associations including Properties/Taxons/OptionTypes and Images.
 
-Specific loaders and rake tasks provided out the box for Spree E-Commerce,  
-enabling import/export of all data including Products with complex associations such as Options/Variants and Images.
+Loaders can be configured via YAML with over ride values, default values and mandatory column settingss.
 
-Many example Spreadsheets in spec/fixtures, fully documented with comments for each column.
+Many example Spreadsheets/CSV files in spec/fixtures, fully documented with comments for each column.
 
 ## Installation
 

data/VERSION

@@ -1 +1 @@
-0.6.0
+0.6.1

data/datashift.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "datashift"
-  s.version = "0.6.0"
+  s.version = "0.6.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Thomas Statter"]
-  s.date = "2012-05-18"
+  s.date = "2012-05-24"
   s.description = "A suite of tools to move data between ActiveRecord models,databases,applications like Excel/Open Office, files and projects including Spree"
   s.email = "rubygems@autotelik.co.uk"
   s.extra_rdoc_files = [
@@ -151,7 +151,6 @@ Gem::Specification.new do |s|
     "tasks/file_tasks.rake",
     "tasks/import/csv.rake",
     "tasks/import/excel.rake",
-    "tasks/spree/image_load.rake",
     "tasks/word_to_seedfu.rake",
     "test/helper.rb",
     "test/test_interact.rb"

data/lib/loaders/excel_loader.rb

@@ -124,6 +124,8 @@ module DataShift
           end
         end
         puts "Excel loading stage complete - #{loaded_objects.size} rows added."
+        
+        puts "WARNING : #{failed_objects.size} rows contained errors and #{failed_objects.size} records NOT created." unless failed_objects.empty?
       end
 
       def value_at(row, column)

data/lib/loaders/loader_base.rb

@@ -100,6 +100,7 @@ module DataShift
       
       @method_mapper = DataShift::MethodMapper.new
       @options = options.clone
+      @verbose_logging = @options[:verbose_logging]
       @headers = []
 
       @default_data_objects ||= {}
@@ -216,7 +217,11 @@ module DataShift
     
     # Default values can be provided in YAML config file
     #  Format :
-    #    Load Class
+    #  
+    #    LoaderClass:
+    #     option: value
+    #     
+    #    Load Class:    (e.g Spree:Product)
     #         atttribute: value
     
     def configure_from( yaml_file )
@@ -251,6 +256,8 @@ module DataShift
       
       if(data[load_object_class.name])
         
+        logger.info("Assigning defaults and over rides from config")
+        
         deflts = data[load_object_class.name]['datashift_defaults']
         @default_values.merge!(deflts) if deflts
         
@@ -258,6 +265,11 @@ module DataShift
         @override_values.merge!(ovrides) if ovrides
       end
       
+      if(data['LoaderBase'])
+        @options.merge!(data['LoaderBase'])
+      end
+      
+      logger.info("Loader Options : #{@options.inspect}")
     end
     
     # Set member variables to hold details and value.
@@ -285,13 +297,28 @@ module DataShift
       @current_value
     end
     
-
+    # return the find_by operator and the values to find
+    def get_find_operator_and_rest( column_data)
+    
+      find_operator, col_values = "",nil
+           
+      if(@current_method_detail.find_by_operator)
+        find_operator, col_values = @current_method_detail.find_by_operator, column_data
+      else
+        find_operator, col_values = column_data.split(LoaderBase::name_value_delim) 
+      end
+       
+      return find_operator, col_values
+    end
+    
     # Process a value string from a column.
     # Assigning value(s) to correct association on @load_object.
     # Method detail represents a column from a file and it's correlated AR associations.
     # Value string which may contain multiple values for a collection association.
     #
-    def process()
+    def process()  
+      
+      logger.info("Current value to assign : #{@current_value}") #if @options['verboose_logging']
       
       if(@current_method_detail.operator_for(:has_many))
 
@@ -310,17 +337,18 @@ module DataShift
 
           columns.each do |col_str|
             
-            find_operator, col_values = "",""
-           
-            if(@current_method_detail.find_by_operator)
-              find_operator, col_values = @current_method_detail.find_by_operator, col_str
-            else
-              find_operator, col_values = col_str.split(LoaderBase::name_value_delim)
-              raise "No key to find #{@current_method_detail.operator} in DB. Expected format key:value" unless(col_values)
-            end
+            find_operator, col_values = get_find_operator_and_rest( col_str )
             
+            #if(@current_method_detail.find_by_operator)
+            #   find_operator, col_values = @current_method_detail.find_by_operator, col_str
+            #  else
+            #    find_operator, col_values = col_str.split(LoaderBase::name_value_delim) 
+            #  end
+          
+            raise "Cannot perform DB find by #{find_operator}. Expected format key:value" unless(find_operator && col_values)
+             
             find_by_values = col_values.split(LoaderBase::multi_value_delim)
-
+                     
             if(find_by_values.size > 1)
 
               @current_value = @current_method_detail.operator_class.send("find_all_by_#{find_operator}", find_by_values )

data/lib/loaders/spree/product_loader.rb

@@ -215,8 +215,10 @@ module DataShift
       
       # Special case for ProductProperties since it can have additional value applied.
       # A list of Properties with a optional Value - supplied in form :
-      #   property.name:value|property.name|property.name:value
-      #
+      #   property_name:value|property_name|property_name:value
+      #  Example :
+      #  test_pp_002|test_pp_003:Example free value|yet_another_property
+
       def add_properties
         # TODO smart column ordering to ensure always valid by time we get to associations
         save_if_new
@@ -224,27 +226,31 @@ module DataShift
         property_list = get_each_assoc#current_value.split(LoaderBase::multi_assoc_delim)
 
         property_list.each do |pstr|
-          pname, pvalue = pstr.split(LoaderBase::name_value_delim)
-          property = @@property_klass.find_by_name(pname)
+          
+        find_by_name, find_by_value = get_find_operator_and_rest( pstr )
+                 
+        raise "Cannot find Property via #{find_by_name} (with value #{find_by_value})" unless(find_by_name)
+                        
+        property = @@property_klass.find_by_name(find_by_name)
 
           unless property
-            property = @@property_klass.create( :name => pname, :presentation => pname.humanize)
-            logger.debug "Created New Property #{property.inspect}"
+            property = @@property_klass.create( :name => find_by_name, :presentation => find_by_name.humanize)
+            logger.info "Created New Property #{property.inspect}"
           end
 
           if(property)
             if(SpreeHelper::version.to_f >= 1.1)
               # Property now protected from mass assignment 
-              x = @@product_property_klass.new( :value => pvalue )
+              x = @@product_property_klass.new( :value => find_by_value )
               x.property = property
               x.save
-              logger.debug "Created New ProductProperty #{x.inspect}"
+              logger.info "Created New ProductProperty #{x.inspect}"
               @load_object.product_properties << x 
             else
-              @load_object.product_properties << @@product_property_klass.create( :property => property, :value => pvalue)
+              @load_object.product_properties << @@product_property_klass.create( :property => property, :value => find_by_values)
             end
           else
-            puts "WARNING: Property #{pname} NOT found - Not set Product"
+            puts "WARNING: Property #{find_by_name} NOT found - Not set Product"
           end
          
         end

data/lib/thor/spree/bootstrap_cleanup.thor

@@ -16,27 +16,37 @@ module Datashift
        
     desc "cleanup", "Remove Spree Product/Variant data from DB"
     
-    def cleanup()  #, [:input, :verbose, :sku_prefix] => :environment do |t, args|
+    def cleanup()
 
-     require 'spree_helper'
+      require 'spree_helper'
 
       require File.expand_path('config/environment.rb')
-    
-      %w{Image OptionType OptionValue Product Property ProductGroup ProductProperty ProductOptionType Variant Taxonomy Taxon Zone}.each do |k|
-        instance_variable_set("@#{k}_klass", DataShift::SpreeHelper::get_spree_class(k)) 
-        puts "Clearing model #{DataShift::SpreeHelper::get_spree_class(k)}"
-        instance_variable_get("@#{k}_klass").delete_all
+
+      cleanup =  %w{ Image OptionType OptionValue 
+                    Product Property ProductGroup ProductProperty ProductOptionType 
+                    Variant Taxonomy Taxon Zone
+      }
+
+      cleanup.each do |k|
+        klass = DataShift::SpreeHelper::get_spree_class(k)
+      if(klass)
+        puts "Clearing model #{klass}"
+        klass.delete_all
+      else
+        puts "WARNING - Coulod not find AR model for class name #{k}"
       end
+        
+    end
       
-      if(File.exists?('public/spree/products') )
-        puts "Removing old Product assets from 'public/spree/products'"
-        FileUtils::rm_rf('public/public/spree/products') 
-      end
+    if(File.exists?('public/spree/products') )
+      puts "Removing old Product assets from 'public/spree/products'"
+      FileUtils::rm_rf('public/public/spree/products') 
+    end
       
-      FileUtils::rm_rf('MissingRecords') if(File.exists?('MissingRecords') )
+    FileUtils::rm_rf('MissingRecords') if(File.exists?('MissingRecords') )
       
-    end
-  
   end
+  
+end
 
 end

data/lib/thor/spree/products_images.thor

@@ -75,24 +75,27 @@ module Datashift
     method_option :sku_prefix, :aliases => '-p', :desc => "Prefix to add to each SKU in import file"
     method_option :dummy, :aliases => '-d', :type => :boolean, :desc => "Dummy run, do not actually save Image or Product"
     method_option :verbose, :aliases => '-v', :type => :boolean, :desc => "Verbose logging"
-    #method_option :config, :aliases => '-c',  :type => :string, :desc => "Configuration file containg defaults or over rides in YAML"
+    method_option :config, :aliases => '-c',  :type => :string, :desc => "Configuration file for Image Loader in YAML"
+    
    
     def images()
 
       require File.expand_path('config/environment.rb')
       
       require 'image_loader'
-
-      @image_cache = options[:input]
        
       puts "Using Product Name for lookup" unless(options[:sku])
       puts "Using SKU for lookup" if(options[:sku])
-         
-     
+           
       attachment_klazz  = DataShift::SpreeHelper::get_spree_class('Product' )
-      sku_klazz         = DataShift::SpreeHelper::get_spree_class('Variant' )
+      attachment_field  = 'name'
       image_klazz       = DataShift::SpreeHelper::get_spree_class('Image' )
 
+      if(options[:sku])
+        attachment_klazz =  DataShift::SpreeHelper::get_spree_class('Variant' ) 
+        attachment_field = 'sku'
+      end
+      
       # TODO generalise for any paperclip project, for now just Spree
       #begin
       #  attachment_klazz = Kernel.const_get(args[:model]) if(args[:model])
@@ -102,6 +105,22 @@ module Datashift
 
       image_loader = DataShift::SpreeHelper::ImageLoader.new
 
+      @loader_config = {}
+      
+      if(options[:config])
+        raise "Bad Config - Cannot find specified file #{options[:config]}" unless File.exists?(options[:config])
+        
+        image_loader.configure_from( options[:config] )
+        
+        @loader_config = YAML::load( File.open(options[:config]) )
+        
+        @loader_config = @loader_config['ImageLoader']
+      end
+      
+      puts "CONFIG: #{@loader_config.inspect}"
+      
+      @image_cache = options[:input]
+      
       if(File.directory? @image_cache )
         logger.info "Loading Spree images from #{@image_cache}"
 
@@ -109,32 +128,23 @@ module Datashift
         Dir.glob("#{@image_cache}/**/*.{jpg,png,gif}") do |image_name|
 
           base_name = File.basename(image_name, '.*')
-           
-          logger.info "Processing #{base_name} : #{File.exists?(image_name)}"
+          base_name.strip!
+                       
+          logger.info "Processing image #{base_name} : #{File.exists?(image_name)}"
            
           record = nil
-          if(options[:sku])
-            sku = base_name.slice!(/\w+/)
-            sku.strip!
-            base_name.strip!
-            
-            sku = "#{options[:sku_prefix]}#{sku}" if(options[:sku_prefix])
-
-            record = sku_klazz.find_by_sku(sku)
-            
-            unless record   # try splitting up filename in various ways looking for the SKU
-              sku.split( '_' ).each do |x| 
-                x = "#{options[:sku_prefix]}#{x}" if(options[:sku_prefix])
-                record = sku_klazz.find_by_sku(x)
-                break if record
-              end
-            end
-            
-            record = record.product if(record)  # SKU stored on Variant but we want it's master Product
-            
-          else
-            record = attachment_klazz.find_by_name(base_name)
+           
+          put "Processing image [#{base_name}]"
+           
+          # unless record   # try splitting up filename in various ways looking for the SKU
+          split_on = @loader_config[:split_file_name_on] || '_'
+              
+          base_name.split(split_on).each do |x| 
+            record = get_record_by(attachment_klazz, attachment_field, x)
+            break if record
           end
+            
+          record = record.product if(record)  # SKU stored on Variant but we want it's master Product
       
           if(record)
             logger.info "Found record for attachment : #{record.inspect}"
@@ -175,6 +185,20 @@ module Datashift
         exit(-1)
       end
     end
+   
+    private
+    
+    def get_record_by(klazz, field, value)
+      x =  (options[:sku_prefix]) ? "#{options[:sku_prefix]}#{value}" : value
+
+      if(@loader_config['case_sensitive'])
+        puts "Search case sensitive for [#{x}] on #{field}"
+         return klazz.find(:first, :conditions => [ "? = ?", field, x ])
+      else
+        puts "Search for [#{x}] on #{field}"
+        return klazz.find(:first, :conditions => [ "lower(?) = ?", field, x.downcase ])
+      end
+    end
   end
 
 end

data/spec/spree_loader_spec.rb

@@ -83,11 +83,11 @@ describe 'SpreeLoader' do
 
   # Loader should perform identically regardless of source, whether csv, .xls etc
   
-  it "should load basic Products .xls via Spree loader" do
+  it "should load basic Products .xls via Spree loader", :fail => true do
     test_basic_product('SpreeProductsSimple.xls')
   end
 
-  it "should load basic Products from .csv via Spree loader", :csv => true do
+  it "should load basic Products from .csv via Spree loader", :csv => true, :fail => true do
     test_basic_product('SpreeProductsSimple.csv')
   end
 
@@ -104,6 +104,10 @@ describe 'SpreeLoader' do
     @product_loader.perform_load( SpecHelper::spree_fixture(source), :mandatory => ['sku', 'name', 'price'] )
 
     @Product_klass.count.should == 3
+    
+    # 2 products available_on set in past, 1 in future
+    @Product_klass.active.size.should == 2
+    @Product_klass.available.size.should == 2
 
     @product_loader.failed_objects.size.should == 0
     @product_loader.loaded_objects.size.should == 3
@@ -179,7 +183,7 @@ describe 'SpreeLoader' do
   end
 
   
-  it "should load Products and create Variants from multiple column", :fail => true do
+  it "should load Products and create Variants from multiple column #{SpecHelper::spree_fixture('SpreeProductsMultiColumn.xls')}", :fail => true do
     test_variants_creation('SpreeProductsMultiColumn.xls')
   end
   

data/spec/thor_spec.rb

@@ -28,5 +28,8 @@ describe 'Thor high level command line tasks' do
     Datashift::Spree.start(["products"])
   end
 
-    
+  it "should be able to run an export to excel from a simple command line task" do
+    Datashift::Spree.start(["products"])
+  end
+  
 end

metadata

@@ -2,7 +2,7 @@
 name: datashift
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.6.0
+  version: 0.6.1
 platform: ruby
 authors: 
   - Thomas Statter
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-05-18 00:00:00 Z
+date: 2012-05-24 00:00:00 Z
 dependencies: 
   - !ruby/object:Gem::Dependency 
     name: spreadsheet
@@ -167,7 +167,6 @@ files:
   - tasks/file_tasks.rake
   - tasks/import/csv.rake
   - tasks/import/excel.rake
-  - tasks/spree/image_load.rake
   - tasks/word_to_seedfu.rake
   - test/helper.rb
   - test/test_interact.rb

data/tasks/spree/image_load.rake

@@ -1,108 +0,0 @@
-# Copyright:: (c) Autotelik Media Ltd 2011
-# Author ::   Tom Statter
-# Date ::     Feb 2011
-# License::   MIT
-#
-# Usage::
-#
-# => rake datashift:spree:images input=vendor/extensions/site/fixtures/images
-# => rake datashift:spree:images input=C:\images\photos large dummy=true
-#
-# => rake datashift:spree:images input=C:\images\taxon_icons skip_if_no_assoc=true klass=Taxon
-#
-namespace :datashift do
-
-  namespace :spree do
-
-    #DEPRECATED FOTR THOR
-    #desc "Populate the DB with images.\nDefault location db/image_seeds, or specify :input=<path> or dir under db/image_seeds with :folder"
-    # :dummy => dummy run without actual saving to DB
-    task :images, [:input, :folder, :dummy, :sku, :skip_if_no_assoc, :skip_if_loaded, :model] => :environment do |t, args|
-
-      require 'image_loader'
-
-      raise "USAGE: Please specify one of :input or :folder" if(args[:input] && args[:folder])
-      puts  "SKU not specified " if(args[:input] && args[:folder])
-
-      if args[:input]
-        @image_cache = args[:input]
-      else
-        @image_cache =  File.join(Rails.root, "db", "image_seeds")
-        @image_cache =  File.join(@image_cache, args[:folder]) if(args[:folder])
-      end
-
-      
-      attachment_klazz = SpreeHelper::get_spree_class('Product' )
-      sku_klazz = SpreeHelper::get_spree_class('Variant' )
-
-      image_loader = ImageLoader.new
-      
-      
-      if(File.directory? @image_cache )
-        puts "Loading images from #{@image_cache}"
-
-        missing_records = []
-        Dir.glob("#{@image_cache}/**/*.{jpg,png,gif}") do |image_name|
-
-          base_name = File.basename(image_name, '.*')
-           
-          puts "Processing #{base_name} : #{File.exists?(image_name)}"
-           
-          record = nil
-          if(args[:sku])
-            sku = base_name.slice!(/\w+/)
-            sku.strip!
-            base_name.strip!
-
-            puts "Looking fo SKU #{sku}"
-            record = sku_klazz.find_by_sku(sku)
-            if record
-              record = record.product   # SKU stored on Variant but we want it's master Product
-            else
-              puts "Looking for NAME [#{base_name}]"
-              record = attachment_klazz.find_by_name(base_name)
-            end
-          else
-            puts "Looking for #{attachment_klazz.name} with NAME [#{base_name}]"
-            record = attachment_klazz.find_by_name(base_name)
-          end
-      
-          if(record)
-            puts "Found record for attachment : #{record.inspect}"
-            exists = record.images.detect {|i| puts "COMPARE #{i.attachment_file_name} => #{image_name}"; i.attachment_file_name == image_name }
-            puts "Found existing attachments [#{exists}]" unless(exists.nil?)
-            if(args[:skip_if_loaded] && !exists.nil?)
-              puts "Skipping - Image #{image_name} already loaded for #{attachment_klazz}"
-              next
-            end
-          else
-            missing_records << image_name
-          end
-
-          # Now do actual upload to DB unless we are doing a dummy run,
-          # or the Image must have an associated record
-          unless(args[:dummy] == 'true' || (args[:skip_if_no_assoc] && record.nil?))
-            image_loader.reset()
-            puts "Process Image"
-            image_loader.process( image_name, record )
-          end
-
-        end
-
-        unless missing_records.empty?
-          FileUtils.mkdir_p('MissingRecords') unless File.directory?('MissingRecords')
-        
-          puts '\nMISSING Records Report>>'
-          missing_records.each do |i|
-            puts "Copy #{i} to MissingRecords folder"
-            FileUtils.cp( i, 'MissingRecords')  unless(args[:dummy] == 'true')
-          end
-        end
-
-      else
-        puts "ERROR: Supplied Path #{@image_cache} not accesible"
-        exit(-1)
-      end
-    end
-  end
-end