datashift 0.9.0 → 0.10.0

data/lib/thor/spree/bootstrap_cleanup.thor

@@ -1,61 +0,0 @@
-# Copyright:: (c) Autotelik Media Ltd 2012
-# Author ::   Tom Statter
-# Date ::     March 2012
-# License::   MIT. Free, Open Source.
-#
-# Usage::
-# bundle exec thor help datashift:spreeboot
-# bundle exec thor datashift:spreeboot:cleanup
-#
-# Note, not DataShift, case sensitive, create namespace for command line : datashift
-
-module Datashift
-        
-  class Spreeboot < Thor     
-  
-    include DataShift::Logging
-       
-    desc "cleanup", "Remove Spree Product/Variant data from DB"
-    
-    def cleanup()
-
-      require File.expand_path('config/environment.rb')
-
-      require 'spree_helper'
-
-      require File.expand_path('config/environment.rb')
-
-      ActiveRecord::Base.connection.execute("TRUNCATE spree_products_taxons")
-      
-      cleanup =  %w{ Image OptionType OptionValue 
-                    Product Property ProductGroup ProductProperty ProductOptionType 
-                    Variant Taxonomy Taxon
-      }
-
-      cleanup.each do |k|
-        klass = DataShift::SpreeHelper::get_spree_class(k)
-        if(klass)
-          puts "Clearing model #{klass}"
-          begin
-            klass.delete_all
-          rescue => e
-            puts e
-          end
-        else
-          puts "WARNING - Could not find AR model for class name #{k}"
-        end
-      end
-      
-      image_bank = 'public/spree/products'
-      
-      if(File.exists?(image_bank) )
-        puts "Removing old Product assets from '#{image_bank}'"
-        FileUtils::rm_rf(image_bank) 
-      end
-      
-      FileUtils::rm_rf('MissingRecords') if(File.exists?('MissingRecords') )
-      
-    end
-  
-  end
-end

data/lib/thor/spree/products_images.thor

@@ -1,252 +0,0 @@
-# Copyright:: (c) Autotelik Media Ltd 2012
-# Author ::   Tom Statter
-# Date ::     March 2012
-# License::   MIT. Free, Open Source.
-#
-# Usage::
-# bundle exec thor help datashift:spree
-# bundle exec thor datashift:spree:products -i db/datashift/MegamanFozz20111115_load.xls -s 299S_
-#
-# bundle exec thor  datashift:spree:images -i db/datashift/imagebank -s -p 299S_
-#
-
-# Note, not DataShift, case sensitive, create namespace for command line : datashift
-module Datashift
-  
-        
-  class Spree < Thor     
-  
-    include DataShift::Logging
-       
-    desc "products", "Populate Spree Product/Variant data from .xls (Excel) or CSV file"
-     
-    method_option :input, :aliases => '-i', :required => true, :desc => "The import file (.xls or .csv)"
-    method_option :sku_prefix, :aliases => '-s', :desc => "Prefix to add to each SKU before saving Product"
-    method_option :verbose, :aliases => '-v', :type => :boolean, :desc => "Verbose logging"
-    method_option :config, :aliases => '-c',  :type => :string, :desc => "Configuration file containg defaults or over rides in YAML"
-    
-    def products()
-
-      # TODO - We're assuming run from a rails app/top level dir...
-      # ...can we make this more robust ? e.g what about when using active record but not in Rails app, 
-      require File.expand_path('config/environment.rb')
-      
-      input = options[:input]
-
-      require 'product_loader'
-
-      loader = DataShift::SpreeHelper::ProductLoader.new
-
-      # YAML configuration file to drive defaults etc
-
-      if(options[:config])
-        raise "Bad Config - Cannot find specified file #{options[:config]}" unless File.exists?(options[:config])
-        
-        loader.configure_from( options[:config] )
-      else
-        loader.set_default_value('available_on', Time.now.to_s(:db) )
-        loader.set_default_value('cost_price', 0.0 )
-        loader.set_default_value('price', 0.0 )
-      end
-      
-      loader.set_prefix('sku', options[:sku_prefix] ) if(options[:sku_prefix])
-      
-      puts "DataShift::Product starting upload from file: #{input}"
-
-      options = {:mandatory => ['sku', 'name', 'price']}
-    
-      # In >= 1.1.0 Image moved to master Variant from Product
-      options[:force_inclusion] = ['images'] if(DataShift::SpreeHelper::version.to_f > 1 )
-      
-      loader.perform_load(input, options)
-    end
-  
-
-    desc "attach_images", "Populate Products with images from Excel/CSV\nProvide column SKU or Name\nProvide column 'attachment' containing full path to image"
-    # :dummy => dummy run without actual saving to DB
-    method_option :input, :aliases => '-i', :required => true, :desc => "The 2 column import file (.xls or .csv)"
-    
-    def attach_images()
-
-      require File.expand_path('config/environment.rb')
-      
-      require 'image_loader'
-      
-      image_klazz = DataShift::SpreeHelper::get_spree_class('Image' )
-      
-      # force inclusion means add to operator list even if not present
-      options = { :force_inclusion => ['sku', 'attachment'] } if(SpreeHelper::version.to_f > 1 )
-    
-      loader = DataShift::SpreeHelper::ImageLoader.new(nil, options)
-    
-      loader.perform_load( options[:input], options )
-    end
-  
-    
-    #
-    # => thor datashift:spree:images input=vendor/extensions/site/fixtures/images
-    # => rake datashift:spree:images input=C:\images\photos large dummy=true
-    #
-    # => rake datashift:spree:images input=C:\images\taxon_icons skip_if_no_assoc=true klass=Taxon
-    #
-    desc "images", "Populate the DB with images from a directory\nwhere image names contain somewhere the Product Sku/Name"
-    
-    # :dummy => dummy run without actual saving to DB
-    method_option :input, :aliases => '-i', :required => true, :desc => "The input path containing images "
-    
-    method_option :glob, :aliases => '-g',  :desc => 'The glob to use to find files e.g. \'{*.jpg,*.gif,*.png}\' '
-    method_option :recursive, :aliases => '-r', :type => :boolean, :desc => "Scan sub directories of input for images"
-     
-    method_option :sku, :aliases => '-s', :desc => "Lookup Product based on image name starting with sku"
-    method_option :sku_prefix, :aliases => '-p', :desc => "SKU prefix to add to each image name before attempting Product lookup"
-    method_option :dummy, :aliases => '-d', :type => :boolean, :desc => "Dummy run, do not actually save Image or Product"
-    
-    method_option :process_when_no_assoc, :aliases => '-f', :type => :boolean, :desc => "Process image even if no Product found - force loading"
-    method_option :skip_when_assoc, :aliases => '-x', :type => :boolean, :desc => "DO not process image if Product already has image"
-    
-    method_option :verbose, :aliases => '-v', :type => :boolean, :desc => "Verbose logging"
-    method_option :config, :aliases => '-c',  :type => :string, :desc => "Configuration file for Image Loader in YAML"
-    method_option :split_file_name_on,  :type => :string, :desc => "delimiter to progressivley split filename for Prod lookup", :default => '_'
-    method_option :case_sensitive, :type => :boolean, :desc => "Use case sensitive where clause to find Product"
-    method_option :use_like, :type => :boolean, :desc => "Use sku/name LIKE 'string%' instead of sku/name = 'string' in where clauses to find Product"
-  
-    def images()
-
-      require File.expand_path('config/environment.rb')
-      
-      require 'spree/image_loader'
-            
-      @verbose = options[:verbose]
-       
-      puts "Using Product Name for lookup" unless(options[:sku])
-      puts "Using SKU for lookup" if(options[:sku])
-       
-      image_klazz = DataShift::SpreeHelper::get_spree_class('Image' )
-       
-      attachment_klazz  = DataShift::SpreeHelper::get_spree_class('Product' )
-      attachment_field  = 'name'
-
-      if(options[:sku] || SpreeHelper::version.to_f > 1)
-        attachment_klazz =  DataShift::SpreeHelper::get_spree_class('Variant' ) 
-        attachment_field = 'sku'
-      end
-
-      image_loader = DataShift::SpreeHelper::ImageLoader.new(nil, options)
- 
-      if(options[:config])
-        raise "Bad Config - Cannot find specified file #{options[:config]}" unless File.exists?(options[:config])
-        
-        image_loader.configure_from( options[:config] )
-      end
-
-      loader_config = image_loader.options
- 
-      puts "CONFIG: #{loader_config.inspect}"
-      puts "OPTIONS #{options.inspect}"
-      
-      @image_path = options[:input]
-      
-      unless(File.exists?(@image_path))
-        puts "ERROR: Supplied Path [#{@image_path}] not accesible"
-        exit(-1)
-      end
-      
-      logger.info "Loading Spree images from #{@image_path}"
-
-      missing_records = []
-         
-      # try splitting up filename in various ways looking for the SKU
-      split_on = loader_config['split_file_name_on'] || options[:split_file_name_on]
-             
-      image_cache = DataShift::ImageLoading::get_files(@image_path, options)
-      
-      puts "Found #{image_cache.size} image files - splitting names on delimiter : #{split_on}"
-         
-      image_cache.each do |image_name|
-
-        image_base_name = File.basename(image_name)
-
-        logger.info "Processing image file #{image_base_name} "
-          
-        base_name = File.basename(image_name, '.*')
-        base_name.strip!
-            
-        record = nil
-                   
-        record = image_loader.get_record_by(attachment_klazz, attachment_field, base_name)
-         
-        # TODO move into the ImageLoading module 
-        # 
-        # try the seperate individual portions of the filename, front -> back
-        base_name.split(split_on).each do |x|
-          x = "#{options[:sku_prefix]}#{x}" if(options[:sku_prefix])
-         
-          record = image_loader.get_record_by(attachment_klazz, attachment_field, x)
-          break if record
-        end unless(record)
-            
-        # this time try sequentially and incrementally scanning
-        base_name.split(split_on).inject("") do |str, x|
-          z = (options[:sku_prefix]) ? "#{options[:sku_prefix]}#{str}#{x}": "#{str}#{x}"
-          puts z
-          record = image_loader.get_record_by(attachment_klazz, attachment_field, z)
-          break if record
-          x
-        end unless(record)
-        
-        # END TODO
-          
-        record = record.product if(record && record.respond_to?(:product))  # SKU stored on Variant but we want it's master Product
-      
-        if(record)
-          logger.info "Found record for attachment : #{record.inspect}"
-    
-          if(options[:skip_when_assoc])
-            
-            paper_clip_name = image_base_name.gsub(Paperclip::Attachment::default_options[:restricted_characters], '_')
-            
-            exists = record.images.detect {|i| puts "Check #{paper_clip_name} matches #{i.attachment_file_name}"; i.attachment_file_name == paper_clip_name }
-            if(exists)
-              rid = record.respond_to?(:name) ? record.name : record.id
-              puts "Skipping Image #{image_name} already loaded for #{rid}"
-              logger.info "Skipping - Image #{image_name} already loaded for #{attachment_klazz}"
-              next 
-            end
-          end
-        else
-          missing_records << image_name
-        end
-          
-        next if(options[:dummy]) # Don't actually create/upload to DB if we are doing dummy run
-
-        # Check if Image must have an associated record
-        if(record || (record.nil? && options[:process_when_no_assoc]))
-          image_loader.reset()
-          
-          logger.info("Adding Image #{image_name} to Product #{record.name}")
-          image_loader.create_image(image_klazz, image_name, record)
-          puts "Added Image #{File.basename(image_name)} to Product #{record.sku} : #{record.name}" if(@verbose)
-        end
-
-      end
-
-      unless missing_records.empty?
-        FileUtils.mkdir_p('MissingRecords') unless File.directory?('MissingRecords')
-        
-        puts "WARNING : #{missing_records.size} of #{image_cache.size} images could not be attached to a Product"
-        puts 'For your convenience a copy of images with MISSING Products will be saved to :  ./MissingRecords'
-        missing_records.each do |i|
-          puts "Copying #{i} to MissingRecords folder" if(options[:verbose])
-          FileUtils.cp( i, 'MissingRecords')  unless(options[:dummy] == 'true')
-        end
-      else
-        puts "All images (#{image_cache.size}) were succesfully attached to a Product"
-      end
-
-      puts "Dummy Run Complete- if happy run without -d" if(options[:dummy])
-   
-    end
-   
-  end
-
-end

data/lib/thor/spree/reports.thor

@@ -1,56 +0,0 @@
-# Copyright:: (c) Autotelik Media Ltd 2012
-# Author ::   Tom Statter
-# Date ::     March 2012
-# License::   MIT. Free, Open Source.
-#
-# Usage::
-# bundle exec thor help datashift:reports:missing_images
-# bundle exec thor datashift:spreeboot:cleanup
-#
-# Note, not DataShift, case sensitive, create namespace for command line : datashift
-
-require 'excel_exporter'
-  
-module Datashift
-        
-    class Spreereports < Thor     
-  
-      include DataShift::Logging
-       
-      desc "no_image", "Spree Products without an image"
-    
-      def no_image(report = nil)
-
-        require 'spree_helper'
-        require 'csv_exporter'
-        require 'image_loader'
-
-        require File.expand_path('config/environment.rb')
-
-        klass = DataShift::SpreeHelper::get_spree_class('Product')
-      
-        missing = klass.all.find_all {|p| p.images.size == 0 }
-      
-        puts "There are #{missing.size} Products (of #{klass.count}) without an associated Image"
-      
-        fname = report ? report : "missing_images"
-      
-        if(DataShift::Guards::jruby?)
-          puts "Creating report #{fname}.xls"  
-          DataShift::ExcelExporter.new( fname + '.xls' ).export( missing, :methods => ['sku'] )
-        else
-          puts "Creating report #{fname}.csv"
-          DataShift::CsvExporter.new( fname + '.csv' ).export( missing, :methods => ['sku'] )
-          puts missing.collect(&:name).join('\n')
-        end   
-      
-# TODO - cross check file locations for possible candidates 
-        #image_cache = DataShift::ImageLoading::get_files(@cross_check_location, options)
-        
-        # missing.each { 
-      
-        # puts images.inspect
-      end
-    end
-
-end

data/spec/fixtures/negative/SpreeProdMiss1Mandatory.csv

@@ -1,4 +0,0 @@
-"Name","Description","Available On","     Price","CostPrice","product_properties","count_on_hand","Taxons","Option Types"
-"Demo Product for AR Loader","blah blah","2011-02-14",400.00,320.00,"name:test_pp_001",12,"Paintings","mime_type:jpeg, PDF,PNG"
-"Demo Excel Load via Jruby","less blah","2011-05-14",100.00,30.00,"name:test_pp_001",5,"Paintings","mime_type:jpeg, PDF,PNG"
-"Demo third row in future","more blah blah","2012-07-01",50.34,23.34,"name:test_pp_002,test_pp_003",23,"Drawings","mime_type:jpeg,PDF,PNG|print_type:black_white,colour"

data/spec/fixtures/negative/SpreeProdMissManyMandatory.csv

@@ -1,4 +0,0 @@
-"SKU","Description","Available On","CostPrice","product_properties","count_on_hand","Taxons","Option Types"
-"DEMO_001","blah blah","2011-02-14",320.00,"name:test_pp_001",12,"Paintings","mime_type:jpeg, PDF,PNG"
-"DEMO_002","less blah","2011-05-14",30.00,"name:test_pp_001",5,"Paintings","mime_type:jpeg, PDF,PNG"
-"DEMO_003","more blah blah","2012-07-01",23.34,"name:test_pp_002,test_pp_003",23,"Drawings","mime_type:jpeg,PDF,PNG|print_type:black_white,colour"

data/spec/fixtures/spree/SpreeMultiVariant.csv

@@ -1,4 +0,0 @@
-"SKU","Name","Description","Available On","     Price","CostPrice","Option Types","Option Types","Variants","count_on_hand"
-"MV_001","Demo Product for AR Loader","blah blah","2011-02-14",399.99,320.00,"mime_type:jpeg, PDF;print_type:colour",,"mime_type:PNG","12|6|7"
-"MV_002","Demo Excel Load via Jruby","less blah","2011-05-14",100.00,30.00,"mime_type:jpeg;print_type:black_white",,"mime_type:PNG;print_type:black_white","5|4"
-"MV_003","Demo third row in future","more blah blah","2012-07-01",50.34,23.34,"mime_type:jpeg;print_type:colour, sepia;size:large|mime_type:PNG","mime_type:PDF|print_type:black_white",,"12|4|7|12"

data/spec/fixtures/spree/SpreeProducts.csv

@@ -1,4 +0,0 @@
-"SKU","Name","Description","Available On","     Price","CostPrice","product_properties","Taxons","OptionTypes","Variants","count_on_hand"
-"DEMO_001","Demo Product for AR Loader","blah blah","2011-02-14",399.99,320.00,"test_pp_001","Paintings|WaterColour",,"mime_type:jpeg,PDF,PNG","12|6|7"
-"DEMO_002","Demo Excel Load via Jruby","less blah","2011-05-14",100.00,30.00,"test_pp_003","Paintings|Oils|Paintings>Nature>Seascape","mime_type:jpeg, PDF,PNG",,"5|4|2"
-"DEMO_003","Demo third row in future","more blah blah","2012-07-01",50.34,23.34,"test_pp_002|test_pp_003:Example free value|yet_another_property","Drawings|Landscape",,"mime_type:jpeg,PDF,PNG|print_type:black_white,colour","12|4|7|12|18"

data/spec/fixtures/spree/SpreeProductsDefaults.yml

@@ -1,15 +0,0 @@
-## YAML Template.
-
-# Format is :
-#  --- !ruby/object:ActiveRecordModel
-#   :datashift_defaults
-#       key: value
-#
-
-Spree::Product: 
-  datashift_defaults: 
-   available_on: Time.now.to_s(:db)
-   cost_price: 1.0 
-   meta_description: 'super duper meta desc.'
-   meta_keywords: 'techno dubstep d&b'
-