datashift 0.8.0 → 0.9.0

data/lib/exporters/excel_exporter.rb

@@ -27,6 +27,8 @@ module DataShift
   
       # Create an Excel file from list of ActiveRecord objects
       def export(records, options = {})
+        
+        raise ArgumentError.new('Please supply array of records to export') unless records.is_a? Array
 
         excel = JExcelFile.new()
 
@@ -35,11 +37,14 @@ module DataShift
         else
           excel.create_sheet( records.first.class.name )
         end
-      
+   
         excel.ar_to_headers(records)
         
         excel.ar_to_xls(records)
 
+              
+        # => :methods => List of methods to additionally export on each record
+         
         excel.save( filename() )
       end
       
@@ -90,7 +95,6 @@ module DataShift
         
           excel.set_headers( headers )
         
-
           row_index = 1
         
           items.each do |datum| 

data/lib/exporters/exporter_base.rb

@@ -10,6 +10,12 @@ module DataShift
 
   class ExporterBase
 
+    attr_accessor :filename
+  
+    def initialize(filename)
+      @filename = filename
+    end
+    
   end
 
 end

data/lib/helpers/spree_helper.rb

@@ -97,9 +97,10 @@ module DataShift
         
         store_path = Dir.pwd
         
-        spree_sanbox_app = File.expand_path('../../../sandbox', __FILE__)
+        spree_sanbox_app = File.expand_path('../../../spec/sandbox', __FILE__)
         
         unless(File.exists?(spree_sanbox_app))
+          puts "Creating new Rails sandbox for Spree : #{spree_sanbox_app}"
           Dir.chdir( File.expand_path( "#{spree_sanbox_app}/..") )
           system('rails new sandbox')
         end

data/lib/loaders/paperclip/image_loader.rb

@@ -13,10 +13,16 @@ module DataShift
  
     include DataShift::Logging
     
+    # Get all image files (based on file extensions) from supplied path.
+    # Options : 
+    #     :glob : The glob to use to find files
+    # =>  :recursive : Descend tree looking for files rather than just supplied path
+    
     def self.get_files(path, options = {})
-      glob = (options['recursive'] || options[:recursive])  ? "**/*.{jpg,png,gif}" : "*.{jpg,png,gif}"
+      glob = options[:glob] ? options[:glob] : image_magik_glob
+      glob = (options['recursive'] || options[:recursive])  ? "**/#{glob}" : glob
       
-      Dir.glob("#{path}/#{glob}")
+      Dir.glob("#{path}/#{glob}", File::FNM_CASEFOLD)
     end
   
     def get_file( attachment_path )
@@ -70,6 +76,30 @@ module DataShift
         puts e.inspect, e.backtrace
       end
     end
+    
+    # Set of file extensions ImageMagik can process so default glob
+    # we use to find image files within directories
+    def self.image_magik_glob
+      @im_glob ||= %w{*.3FR *.AAI *.AI *.ART *.ARW  *.AVI *.AVS *.BGR *.BGRA
+                  *.BIE *.BMP *.BMP2  *.BMP3  *.BRF *.CAL *.CALS *.CANVAS
+                  *.CIN *.CIP *.CLIP *.CMYK *.CMYKA *.CR2 *.CRW *.CUR *.CUT *.DCM *.DCR *.DCX
+                  *.DDS *.DFONT *.DJVU *.DNG  *.DOT *.DPS *.DPX
+                  *.EMF *.EPDF  *.EPI *.EPS *.EPS2 *.EPS3  *.EPSF *.EPSI
+                  *.EPT *.EPT2 *.EPT3 *.ERF *.EXR *.FITS *.FPX  *.FTS *.G3 *.GIF *.GIF87
+                  *.GRAY *.HALD  *.HDR *.HRZ  *.ICB *.ICO *.ICON *.IPL
+                  *.ISOBRL *.J2C *.JBG  *.JBIG *.JNG *.JP2 *.JPC *.JPEG *.JPG *.JPX  *.K25 *.KDC
+                  *.LABEL *.M2V  *.M4V  *.MAC *.MAP  *.MAT *.MATTE *.MIFF *.MNG  *.MONO
+                  *.MOV *.MP4 *.MPC *.MPEG *.MPG *.MRW *.MSL *.MSVG *.MTV *.MVG *.NEF *.ORF *.OTB *.OTF *.PAL *.PALM
+                  *.PAM *.PBM  *.PCD *.PCDS *.PCL *.PCT *.PCX *.PDB *.PDF *.PDFA  *.PEF
+                  *.PES *.PFA *.PFB *.PFM *.PGM  *.PGX *.PICON *.PICT *.PIX *.PJPEG *.PLASMA
+                  *.PNG *.PNG24  *.PNG32 *.PNG8 *.PNM *.PPM *.PS *.PS2 *.PS3 *.PSB *.PSD *.PTIF *.PWP *.RAF *.RAS *.RGB
+                  *.RGBA *.RGBO *.RLA *.RLE *.SCR *.SCT *.SFW *.SGI *.SR2 *.SRF
+                  *.SUN *.SVG *.SVGZ *.TGA *.TIFF *.TIFF64 *.TILE *.TIM *.TTC *.TTF *.UBRL *.UIL *.UYVY *.VDA  *.VICAR
+                  *.VID *.VIFF  *.VST *.WBMP *.WEBP  *.WMF *.WMV *.WMZ *.WPG  *.X3F
+                  *.XBM *.XC *.XCF *.XPM *.XPS *.XV *.XWD *.YCbCr *.YCbCrA *.YUV
+      }
+      "{#{@im_glob.join(',')}}"
+    end
   end
       
 end

data/lib/loaders/spree/product_loader.rb

@@ -97,13 +97,14 @@ module DataShift
               values = current_value.to_s.split(LoaderBase::multi_assoc_delim)
 
               if(@load_object.variants.size == values.size)
-                @load_object.variants.each_with_index {|v, i| v.on_hand = values[i].to_i; v.save; }
+                @load_object.variants.each_with_index {|v, i| v.on_hand = values[i].to_i }
+                @load_object.save
               else
                 puts "WARNING: Count on hand entries did not match number of Variants - None Set"
               end
             end
             
-            # can only set count on hand on Product if no Variants exist, else model throws
+          # Can only set count on hand on Product if no Variants exist, else model throws
             
           elsif(@load_object.variants.size == 0) 
             if(current_value.to_s.include?(LoaderBase::multi_assoc_delim))

data/lib/thor/export.thor

@@ -0,0 +1,111 @@
+# Copyright:: (c) Autotelik Media Ltd 2012
+# Author ::   Tom Statter
+# Date ::     April 2012
+# License::   MIT.
+#
+# Usage::
+#
+#  To pull Datashift commands into your main application :
+#
+#     require 'datashift'
+#
+#     DataShift::load_commands
+#
+#  Cmd Line:
+#
+# => bundle exec thor datashift:export:excel -m <active record class> -r <output_template.xls> -a
+#
+require 'datashift'
+  
+# Note, not DataShift, case sensitive, create namespace for command line : datashift
+module Datashift
+
+          
+  class Export < Thor     
+  
+    include DataShift::Logging
+      
+    desc "excel", "export any active record model (with optional associations)" 
+    method_option :model, :aliases => '-m', :required => true, :desc => "The active record model to export"
+    method_option :result, :aliases => '-r', :required => true, :desc => "Create template of model in supplied file"
+    method_option :assoc, :aliases => '-a', :type => :boolean, :desc => "Include all associations in the template"
+    method_option :exclude, :aliases => '-e',  :type => :array, :desc => "Use with -a : Exclude association types. Any from #{DataShift::MethodDetail::supported_types_enum.to_a.inspect}"
+    
+    def excel()
+     
+      # TODO - We're assuming run from a rails app/top level dir...
+      # ...can we make this more robust ? e.g what about when using active record but not in Rails app, 
+      require File.expand_path('config/environment.rb')
+   
+      require 'excel_exporter'
+
+      model = options[:model]
+      result = options[:result]
+     
+      logger.info "Datashift: Start Excel export to #{result}"
+            
+      klass = ModelMapper::class_from_string(model)  #Kernel.const_get(model)
+    
+      raise "ERROR: No such Model [#{model}] found - check valid model supplied via -model <Class>" if(klass.nil?)
+
+      begin
+        gen = DataShift::ExcelExporter.new(result)
+
+        if(options[:assoc])
+          opts = (options[:exclude]) ? {:exclude => options[:exclude]} : {}
+          logger.info("Datashift: Exporting with associations")
+          gen.export_with_associations(klass, klass.all, opts)
+        else
+          gen.export(klass.all, :sheet_name => klass.name)
+        end
+      rescue => e
+        puts e
+        puts e.backtrace
+        puts "Warning: Error during export, data may be incomplete"
+      end
+
+    end
+  
+    desc "csv", "export any active record model (with optional associations)" 
+    method_option :model, :aliases => '-m', :required => true, :desc => "The active record model to export"
+    method_option :result, :aliases => '-r', :required => true, :desc => "Create template of model in supplied file"
+    method_option :assoc, :aliases => '-a', :type => :boolean, :desc => "Include all associations in the template"
+    method_option :exclude, :aliases => '-e',  :type => :array, :desc => "Use with -a : Exclude association types. Any from #{DataShift::MethodDetail::supported_types_enum.to_a.inspect}"
+    method_option :methods, :aliases => '-c',  :type => :array, :desc => "List of additional methods to call on model, useful for situations like delegated methods"
+    
+    def csv()
+     
+      # TODO - We're assuming run from a rails app/top level dir...
+      # ...can we make this more robust ? e.g what about when using active record but not in Rails app, 
+      require File.expand_path('config/environment.rb')
+   
+      require 'csv_exporter'
+
+      model = options[:model]
+      result = options[:result]
+     
+      logger.info "Datashift: Start CSV export to #{result}"
+            
+      klass = ModelMapper::class_from_string(model)  #Kernel.const_get(model)
+    
+      raise "ERROR: No such Model [#{model}] found - check valid model supplied via -model <Class>" if(klass.nil?)
+
+      begin
+        gen = DataShift::CsvExporter.new(result)
+
+        if(options[:assoc])
+          logger.info("Datashift: Exporting with associations")
+          gen.export_with_associations(klass, klass.all, options)
+        else
+          gen.export(klass.all, options)
+        end
+      rescue => e
+        puts e
+        puts e.backtrace
+        puts "Warning: Error during export, data may be incomplete"
+      end
+
+    end
+  end
+
+end

data/lib/thor/{import_excel.thor → import.thor}

@@ -33,7 +33,8 @@ module Datashift
     method_option :config, :aliases => '-c', :desc => "YAML config file with defaults, over-rides etc"
     method_option :assoc, :aliases => '-a', :type => :boolean, :desc => "Include any associations supplied in the input"
     method_option :exclude, :aliases => '-e',  :type => :array, :desc => "Use with -a : Exclude association types. Any from #{DataShift::MethodDetail::supported_types_enum.to_a.inspect}"
-    
+    method_option :verbose, :aliases => '-v', :type => :boolean, :desc => "Verbose logging"
+     
     def excel()
      
       # TODO - We're assuming run from a rails app/top level dir...
@@ -70,13 +71,50 @@ module Datashift
       end
 
       logger.info("ARGS #{options.inspect}")
-      loader.logger.verbose if(ENV['verbose'])
+      loader.logger.verbose if(options['verbose'])
       
       loader.configure_from( options[:config] ) if(options[:config])
 
 
       loader.perform_load(options[:input])
     end
+    
+    desc "csv", "import CSV file for specified active record model" 
+    method_option :model, :aliases => '-m', :required => true, :desc => "The related active record model"
+    method_option :input, :aliases => '-i', :required => true, :desc => "The input .xls file"
+    method_option :config, :aliases => '-c', :desc => "YAML config file with defaults, over-rides etc"
+    method_option :assoc, :aliases => '-a', :type => :boolean, :desc => "Include any associations supplied in the input"
+    method_option :exclude, :aliases => '-e',  :type => :array, :desc => "Use with -a : Exclude association types. Any from #{DataShift::MethodDetail::supported_types_enum.to_a.inspect}"
+    method_option :verbose, :aliases => '-v', :type => :boolean, :desc => "Verbose logging"
+
+    def csv()
+     
+      # TODO - We're assuming run from a rails app/top level dir...
+      # ...can we make this more robust ? e.g what about when using active record but not in Rails app, 
+      require File.expand_path('config/environment.rb')
+   
+      require 'csv_loader'
+
+      model = options[:model]
+      begin
+        # support modules e.g "Spree::Property") 
+        klass = ModelMapper::class_from_string(model) 
+      rescue NameError
+        raise "ERROR: No such AR Model found - check valid model supplied with -m <Class>"
+      end
+
+      raise "ERROR: No such AR Model found - check valid model supplied with -m <Class>" if(klass.nil?) 
+      
+
+      loader = DataShift::CsvLoader.new(klass)
+      
+      loader.logger.verbose if(options['verbose'])
+      
+      loader.configure_from( options[:config] ) if(options[:config])
+
+      loader.perform_load(options[:input])
+    end
+    
   end
 
 end

data/lib/thor/spree/bootstrap_cleanup.thor

@@ -9,9 +9,6 @@
 #
 # Note, not DataShift, case sensitive, create namespace for command line : datashift
 
-require File.expand_path('config/environment.rb')
-
-
 module Datashift
         
   class Spreeboot < Thor     
@@ -22,6 +19,8 @@ module Datashift
     
     def cleanup()
 
+      require File.expand_path('config/environment.rb')
+
       require 'spree_helper'
 
       require File.expand_path('config/environment.rb')
@@ -37,7 +36,11 @@ module Datashift
         klass = DataShift::SpreeHelper::get_spree_class(k)
         if(klass)
           puts "Clearing model #{klass}"
-          klass.delete_all
+          begin
+            klass.delete_all
+          rescue => e
+            puts e
+          end
         else
           puts "WARNING - Could not find AR model for class name #{k}"
         end

data/lib/thor/spree/products_images.thor

@@ -92,12 +92,13 @@ module Datashift
     desc "images", "Populate the DB with images from a directory\nwhere image names contain somewhere the Product Sku/Name"
     
     # :dummy => dummy run without actual saving to DB
-    method_option :input, :aliases => '-i', :required => true, :desc => "The input path containing images (.jpg, .gif, .png)"
+    method_option :input, :aliases => '-i', :required => true, :desc => "The input path containing images "
     
+    method_option :glob, :aliases => '-g',  :desc => 'The glob to use to find files e.g. \'{*.jpg,*.gif,*.png}\' '
     method_option :recursive, :aliases => '-r', :type => :boolean, :desc => "Scan sub directories of input for images"
      
     method_option :sku, :aliases => '-s', :desc => "Lookup Product based on image name starting with sku"
-    method_option :sku_prefix, :aliases => '-p', :desc => "Prefix to add to each SKU in import file before attempting lookup"
+    method_option :sku_prefix, :aliases => '-p', :desc => "SKU prefix to add to each image name before attempting Product lookup"
     method_option :dummy, :aliases => '-d', :type => :boolean, :desc => "Dummy run, do not actually save Image or Product"
     
     method_option :process_when_no_assoc, :aliases => '-f', :type => :boolean, :desc => "Process image even if no Product found - force loading"
@@ -156,36 +157,44 @@ module Datashift
          
       # try splitting up filename in various ways looking for the SKU
       split_on = loader_config['split_file_name_on'] || options[:split_file_name_on]
-       
-      puts "Will scan image names splitting on delimiter : #{split_on}"
-      
+             
       image_cache = DataShift::ImageLoading::get_files(@image_path, options)
       
+      puts "Found #{image_cache.size} image files - splitting names on delimiter : #{split_on}"
+         
       image_cache.each do |image_name|
 
         image_base_name = File.basename(image_name)
-        
+
+        logger.info "Processing image file #{image_base_name} "
+          
         base_name = File.basename(image_name, '.*')
         base_name.strip!
-                       
-        logger.info "Processing image file #{base_name} : #{File.exists?(image_name)}"
-           
+            
         record = nil
                    
         record = image_loader.get_record_by(attachment_klazz, attachment_field, base_name)
-          
-        # try seperate portions of the filename, front -> back
-        base_name.split(split_on).each do |x| 
+         
+        # TODO move into the ImageLoading module 
+        # 
+        # try the seperate individual portions of the filename, front -> back
+        base_name.split(split_on).each do |x|
+          x = "#{options[:sku_prefix]}#{x}" if(options[:sku_prefix])
+         
           record = image_loader.get_record_by(attachment_klazz, attachment_field, x)
           break if record
         end unless(record)
             
-        # this time try sequentially scanning
-        base_name.split(split_on).inject("") do |str, x| 
-          record = image_loader.get_record_by(attachment_klazz, attachment_field, "#{str}#{x}")
+        # this time try sequentially and incrementally scanning
+        base_name.split(split_on).inject("") do |str, x|
+          z = (options[:sku_prefix]) ? "#{options[:sku_prefix]}#{str}#{x}": "#{str}#{x}"
+          puts z
+          record = image_loader.get_record_by(attachment_klazz, attachment_field, z)
           break if record
           x
         end unless(record)
+        
+        # END TODO
           
         record = record.product if(record && record.respond_to?(:product))  # SKU stored on Variant but we want it's master Product
       
@@ -225,9 +234,9 @@ module Datashift
         FileUtils.mkdir_p('MissingRecords') unless File.directory?('MissingRecords')
         
         puts "WARNING : #{missing_records.size} of #{image_cache.size} images could not be attached to a Product"
-        puts 'Copying all images with MISSING Records to ./MissingRecords >>'
+        puts 'For your convenience a copy of images with MISSING Products will be saved to :  ./MissingRecords'
         missing_records.each do |i|
-          puts "Copy #{i} to MissingRecords folder"
+          puts "Copying #{i} to MissingRecords folder" if(options[:verbose])
           FileUtils.cp( i, 'MissingRecords')  unless(options[:dummy] == 'true')
         end
       else

data/lib/thor/spree/reports.thor

@@ -4,7 +4,7 @@
 # License::   MIT. Free, Open Source.
 #
 # Usage::
-# bundle exec thor help datashift:spreeboot
+# bundle exec thor help datashift:reports:missing_images
 # bundle exec thor datashift:spreeboot:cleanup
 #
 # Note, not DataShift, case sensitive, create namespace for command line : datashift
@@ -13,54 +13,44 @@ require 'excel_exporter'
   
 module Datashift
         
-  class Reports < Thor     
+    class Spreereports < Thor     
   
-    include DataShift::Logging
+      include DataShift::Logging
        
-    desc "missing_images", "Spree Products without an image"
+      desc "no_image", "Spree Products without an image"
     
-    def missing_images(report = nil)
+      def no_image(report = nil)
 
-      require 'spree_helper'
-      require 'image_loader'
+        require 'spree_helper'
+        require 'csv_exporter'
+        require 'image_loader'
 
-      require File.expand_path('config/environment.rb')
+        require File.expand_path('config/environment.rb')
 
-      klass = DataShift::SpreeHelper::get_spree_class('Product')
+        klass = DataShift::SpreeHelper::get_spree_class('Product')
       
-      missing = klass.all.find_all {|p| p.images.size == 0 }
+        missing = klass.all.find_all {|p| p.images.size == 0 }
       
-      puts "There are #{missing.size} Products without an associated Image"
+        puts "There are #{missing.size} Products (of #{klass.count}) without an associated Image"
       
-      if(DataShift::Guards::jruby?)
-        fname = report ? report : "missing_images.xls"
-        DataShift::ExcelExporter.new( fname ).export( missing )
-      else
-        puts missing.collect(&:name).inspect
-      end   
+        fname = report ? report : "missing_images"
       
-      @drop_box = "/home/stattert/Dropbox/DaveWebsiteInfo/"
-
-      @image_list  = %w{
-    010InafixTheArmourGodAllFolders 
-    01Figuresurbanlandscapepaintings 
-    01FinishedArtPrints    
-    02Seascapespainting  
-    03Landscapes 
-    04Spain 
-
-    07SignsJohnAllFolders 
-    07_Mar 
-    09Powerpointsermonaids
-      }
-  
-      options = { :recursive => true }
-    
-      images = @image_list.collect do |p| 
-        DataShift::ImageLoading::get_files(File.join(@drop_box,p), options)
-      end
+        if(DataShift::Guards::jruby?)
+          puts "Creating report #{fname}.xls"  
+          DataShift::ExcelExporter.new( fname + '.xls' ).export( missing, :methods => ['sku'] )
+        else
+          puts "Creating report #{fname}.csv"
+          DataShift::CsvExporter.new( fname + '.csv' ).export( missing, :methods => ['sku'] )
+          puts missing.collect(&:name).join('\n')
+        end   
       
-      puts images.inspect
+# TODO - cross check file locations for possible candidates 
+        #image_cache = DataShift::ImageLoading::get_files(@cross_check_location, options)
+        
+        # missing.each { 
+      
+        # puts images.inspect
+      end
     end
-  end
-end
+
+end