datashift 0.4.3 → 0.5.0

data/VERSION

@@ -1 +1 @@
-0.4.3
+0.5.0

data/datashift.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "datashift"
-  s.version = "0.4.3"
+  s.version = "0.5.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Thomas Statter"]
-  s.date = "2012-03-31"
+  s.date = "2012-05-08"
   s.description = "A suite of tools to move data between ActiveRecord models,databases,applications like Excel/Open Office, files and projects including Spree"
   s.email = "rubygems@autotelik.co.uk"
   s.extra_rdoc_files = [
@@ -24,6 +24,7 @@ Gem::Specification.new do |s|
     "README.rdoc",
     "Rakefile",
     "VERSION",
+    "datashift-0.5.0.gem",
     "datashift.gemspec",
     "lib/applications/jruby/jexcel_file.rb",
     "lib/applications/jruby/word.rb",
@@ -68,6 +69,7 @@ Gem::Specification.new do |s|
     "lib/loaders/spreadsheet_loader.rb",
     "lib/loaders/spree/image_loader.rb",
     "lib/loaders/spree/product_loader.rb",
+    "lib/thor/export_excel.thor",
     "lib/thor/generate_excel.thor",
     "lib/thor/import_excel.thor",
     "lib/thor/spree/bootstrap_cleanup.thor",
@@ -92,6 +94,7 @@ Gem::Specification.new do |s|
     "sandbox/config/database.yml",
     "sandbox/config/environment.rb",
     "sandbox/config/environments/development.rb",
+    "spec/Gemfile",
     "spec/csv_loader_spec.rb",
     "spec/datashift_spec.rb",
     "spec/db/migrate/20110803201325_create_test_bed.rb",
@@ -113,13 +116,13 @@ Gem::Specification.new do |s|
     "spec/fixtures/images/DEMO_002_Powerstation.jpg",
     "spec/fixtures/images/DEMO_003_ror_mug.jpeg",
     "spec/fixtures/images/DEMO_004_ror_ringer.jpeg",
-    "spec/fixtures/interact_models_db.sqlite",
     "spec/fixtures/negative/SpreeProdMiss1Mandatory.csv",
     "spec/fixtures/negative/SpreeProdMiss1Mandatory.xls",
     "spec/fixtures/negative/SpreeProdMissManyMandatory.csv",
     "spec/fixtures/negative/SpreeProdMissManyMandatory.xls",
     "spec/fixtures/simple_export_spec.xls",
     "spec/fixtures/simple_template_spec.xls",
+    "spec/fixtures/spree/SpreeProductImages.xls",
     "spec/fixtures/spree/SpreeProducts.csv",
     "spec/fixtures/spree/SpreeProducts.xls",
     "spec/fixtures/spree/SpreeProductsDefaults.yml",
@@ -142,6 +145,7 @@ Gem::Specification.new do |s|
     "spec/spree_images_loader_spec.rb",
     "spec/spree_loader_spec.rb",
     "spec/spree_method_mapping_spec.rb",
+    "spec/thor_spec.rb",
     "tasks/config/seed_fu_product_template.erb",
     "tasks/config/tidy_config.txt",
     "tasks/export/excel_generator.rake",

data/lib/applications/jruby/jexcel_file.rb

@@ -123,7 +123,7 @@ if(DataShift::Guards::jruby?)
     # Process each row. (type is org.apache.poi.hssf.usermodel.HSSFRow)
 
     def each_row
-      @sheet.rowIterator.each { |row| yield row }
+      @sheet.rowIterator.each { |row| @row = row; yield row }
     end
 
     # Create new row, bring index in line with POI usage (our 1 is their 0)
@@ -150,7 +150,9 @@ if(DataShift::Guards::jruby?)
       create_row(1)
       return if headers.empty?
     
-      set_row(1, 1, headers)
+      headers.each_with_index do |datum, i|
+        @row.createCell(i, excel_cell_type(datum)).setCellValue(datum)
+      end
     end
 
     #  Populate a row  of cells with data in an array 

data/lib/datashift/method_details_manager.rb

@@ -13,10 +13,11 @@ module DataShift
   # Stores MethodDetails for a class mapped by type
   class MethodDetailsManager
 
-    attr_reader :method_details
+    attr_reader :method_details, :method_details_list
+    attr_reader :managed_class_name
     
     def initialize( klass )
-      @parent_class = klass
+      @managed_class_name = klass.name
       @method_details = {}
       @method_details_list = {}
     end
@@ -49,6 +50,10 @@ module DataShift
       @method_details_list[type.to_sym]
     end
     
+    def all_available_operators
+      method_details_list.values.flatten.collect(&:operator)
+    end
+    
   end
   
 end

data/lib/datashift/method_dictionary.rb

@@ -114,13 +114,13 @@ module DataShift
     # If not nil, returned method can be used directly in for example klass.new.send( call, .... )
     #
     def self.find_method_detail( klass, external_name )
-      operator = nil
 
-      # TODO - should we raise error to warn find_operators never called ?
       md_mgr = method_details_mgrs[klass] || MethodDetailsManager.new( klass )
          
+      # md_mgr.all_available_operators.each { |l| puts "DEBUG: Mapped Method : #{l.inspect}" }
+        
       name = external_name.to_s
-
+     
       # TODO - check out regexp to do this work better plus Inflections ??
       # Want to be able to handle any of ["Count On hand", 'count_on_hand', "Count OnHand", "COUNT ONHand" etc]
       [

data/lib/datashift/method_mapper.rb

@@ -61,6 +61,7 @@ module DataShift
         end
         
         x, lookup = name.split(MethodMapper::column_delim) 
+        #puts "DEBUG: Find Method Detail for #{x}"
         md = MethodDictionary::find_method_detail( klass, x )
         
         # TODO be nice if we could cheeck that the assoc on klass responds to the specified

data/lib/exporters/excel_exporter.rb

@@ -67,39 +67,39 @@ module DataShift
         puts "work_list :  [#{work_list.inspect}]"
       
         details_mgr = MethodDictionary.method_details_mgrs[klass]
-                  
+
+                        
+        data = []
+        # For each type belongs has_one, has_many etc find the operators
+        # and create headers, then for each record call those operators
         work_list.each do |op_type|
+          
           list_for_class_and_op = details_mgr.get_list(op_type)
        
           next if(list_for_class_and_op.nil? || list_for_class_and_op.empty?)
-         #if(work_list.include?(md.operator_type))
-              #each do |mdtype|
-          #end
-          #if(MethodDictionary.respond_to?("#{mdtype}_for") )
-           # method_details = MethodDictionary.send("#{mdtype}_for", klass)
+
+          # method_details = MethodDictionary.send("#{mdtype}_for", klass)
         
-            list_for_class_and_op.each {|md| headers << "#{md.operator}" }
-          #else
-           # puts "ERROR : Unknown option in :with [#{mdtype}]"
+          list_for_class_and_op.each do |md| 
+            headers << "#{md.operator}"
+            items.each do |i| 
+              data << i.send( md.operator )
+            end
          
-        end
+          end
         
-        excel.set_headers( headers )
-                
-        data = []
+          excel.set_headers( headers )
         
-        items.each do |record|
-          
-          MethodMapper.method_details[klass].each do |method_detail|   
-            if(method_detail.operator_type == :assignment)
-              data << record.send( method_detail.operator )
-            end
+
+          row_index = 1
+        
+          items.each do |datum| 
+            excel.create_row(row_index += 1)
+            excel.ar_to_xls_row(1, datum)
           end
+           
+          excel.save( filename() )
         end
-        
-        excel.set_row(2,1,items)
-
-        excel.save( filename() )
       end
     end # ExcelGenerator
 

data/lib/helpers/spree_helper.rb

@@ -3,22 +3,19 @@
 # Date ::     Aug 2011
 # License::   MIT
 #
-# Details::   Spree Helper mixing in Support for testing or loading Rails Spree e-commerce.
+# Details::   Spree Helper for Product Loading. 
 # 
-#             The Spree version you want to test should be picked up from the Gemfile
+#             Utils to try to manage different Spree versions seamlessly.
+#             
+#             Spree Helper for RSpec testing, enables mixing in Support for
+#             testing or loading Rails Spree e-commerce.
+# 
+#             The Spree version you want to test should be picked up from spec/Gemfile
 # 
 #             Since datashift gem is not a Rails app or a Spree App, provides utilities to internally
 #             create a Spree Database, and to load Spree components, enabling standalone testing.
 #
-# =>          Has been tested with  0.7
-# 
-# # =>        TODO - Can we move to a Gemfile/bunlder
-#             require 'rubygems'
-#             gemfile = File.expand_path("<%= gemfile_path %>", __FILE__)
-#
-#             ENV['BUNDLE_GEMFILE'] = gemfile
-#             require 'bundler'
-#             Bundler.setup
+# =>          Has been tested with  0.11.2, 0.7, 1.0.0
 #
 # =>          TODO - See if we can improve DB creation/migration ....
 #             N.B Some or all of Spree Tests may fail very first time run,
@@ -52,8 +49,12 @@ module DataShift
       end
     end
     
+    def self.version
+       Gem.loaded_specs['spree'] ? Gem.loaded_specs['spree'].version.version : "0.0.0"
+    end
+    
     def self.is_namespace_version
-      Gem.loaded_specs['spree'].version.version.to_f >= 1
+      SpreeHelper::version.to_f >= 1
     end
   
     def self.lib_root
@@ -79,7 +80,9 @@ module DataShift
     # => Will chdir into the sandbox to load environment as need to mimic being at root of a rails project
     #    chdir back after environment loaded
     
-    def self.boot( database_env )
+    #  gem('rails', '3.2.3')
+    
+    def self.boot( database_env)#, rails_version = '3.1.3' )
      
       if( ! is_namespace_version )
         db_connect( database_env )
@@ -88,14 +91,12 @@ module DataShift
         @dslog.info "Booted Spree using pre 1.0.0 version"
       else
         
-        # TODO as Spree versions moves how do we best track reqd Rails version
-        # parse the gemspec of the core Gemfile ?
-        
-        gem('rails', '3.1.3')
+
+        #em('rails', rails_version)
         
-        db_connect( database_env, '3.1.3' )  
+        db_connect( database_env )#, rails_version )  
         
-        @dslog.info "Booting Spree using post 1.0.0 version"
+        @dslog.info "Booting Spree using version #{SpreeHelper::version}"
        
         require 'rails/all'
         
@@ -111,7 +112,7 @@ module DataShift
         
         Dir.chdir( store_path )
         
-        @dslog.info "Booted Spree using post 1.0.0 version"
+        @dslog.info "Booted Spree using version #{SpreeHelper::version}"
       end
     end
 

data/lib/loaders/csv_loader.rb

@@ -16,6 +16,9 @@ module DataShift
     
     include DataShift::Logging
     
+    # Options :
+    #   strict : Raise exception if any column cannot be mapped
+    
     def perform_csv_load(file_name, options = {})
        
       require "csv"
@@ -32,7 +35,7 @@ module DataShift
       map_headers_to_operators( @parsed_file.shift, options[:strict] , @mandatory )
         
       unless(@method_mapper.missing_methods.empty?)
-        puts "WARNING: Following column headings could not be mapped : #{@method_mapper.missing_methods.inspect}"
+        logger.error("Following column headings could not be mapped :\n#{@method_mapper.missing_methods.inspect}")
         raise MappingDefinitionError, "ERROR: Missing mappings for #{@method_mapper.missing_methods.size} column headings"
       end
 

data/lib/loaders/loader_base.rb

@@ -113,10 +113,29 @@ module DataShift
       reset(object)
     end
 
+    
+    # Based on filename call appropriate loading function
+    # Currently supports :
+    #   Excel/Open Office files saved as .xls
+    #   CSV files
+    # 
+    # OPTIONS :
+    #   strict : Raise exception if any column cannot be mapped
+       
+    def perform_load( file_name, options = {} )
 
-    # kinda the derived classes interface - best way in Ruby ?
-    def perform_load( input, options = {} )
-      raise "WARNING- ABSTRACT METHOD CALLED - Please implement perform_load()"
+      raise DataShift::BadFile, "Cannot load #{file_name} file not found." unless(File.exists?(file_name))
+        
+      ext = File.extname(file_name)
+          
+      if(ext.casecmp('.xls') == 0)
+        raise DataShift::BadRuby, "Please install and use JRuby for loading .xls files" unless(Guards::jruby?)
+        perform_excel_load(file_name, options)
+      elsif(ext.casecmp('.csv') == 0)
+        perform_csv_load(file_name, options)
+      else
+        raise DataShift::UnsupportedFileType, "#{ext} files not supported - Try .csv or OpenOffice/Excel .xls"
+      end
     end
 
     
@@ -126,7 +145,7 @@ module DataShift
     # This is then available through @method_mapper.method_details.each
     # 
     # Options:
-    #  strict : report any header values that can't be mapped as an error
+    #  strict : Raise an exception of any headers can't be mapped to an attribute/association
     #
     def map_headers_to_operators( headers, strict, mandatory = [])
       @headers = headers
@@ -134,6 +153,7 @@ module DataShift
       begin 
         method_details = @method_mapper.map_inbound_to_methods( load_object_class, @headers )
       rescue => e
+        puts e.inspect
         logger.error("Failed to map header row to set of database operators : #{e.inspect}")
         raise MappingDefinitionError, "Failed to map header row to set of database operators"
       end

data/lib/loaders/spree/image_loader.rb

@@ -51,7 +51,9 @@ module DataShift
     class ImageLoader < LoaderBase
 
       include DataShift::ImageLoading
-        
+      include DataShift::CsvLoading
+      include DataShift::ExcelLoading
+      
       def initialize(image = nil)
         puts SpreeHelper::get_spree_class('Image')
         

data/lib/loaders/spree/product_loader.rb

@@ -43,16 +43,20 @@ module DataShift
       # Currently supports :
       #   Excel/Open Office files saved as .xls
       #   CSV files
+      # 
+      # OPTIONS :
+      #   strict : Raise exception if any column cannot be mapped
+       
       def perform_load( file_name, options = {} )
 
         raise DataShift::BadFile, "Cannot load #{file_name} file not found." unless(File.exists?(file_name))
         
         ext = File.extname(file_name)
           
-        if(ext == '.xls')
+        if(ext.casecmp('.xls') == 0)
           raise DataShift::BadRuby, "Please install and use JRuby for loading .xls files" unless(Guards::jruby?)
           perform_excel_load(file_name, options)
-        elsif(ext == '.csv')
+        elsif(ext.casecmp('.csv') == 0)
           perform_csv_load(file_name, options)
         else
           raise DataShift::UnsupportedFileType, "#{ext} files not supported - Try .csv or OpenOffice/Excel .xls"
@@ -182,16 +186,35 @@ module DataShift
             ovname.strip!
             ov = @@option_value_klass.find_or_create_by_name_and_option_type_id(ovname, option_type.id)
             if ov
-              variant = @@variant_klass.create( :product => @load_object, :sku => "#{@load_object.sku}_#{i}", :price => @load_object.price, :available_on => @load_object.available_on)
-              #puts "DEBUG: Created New Variant: #{variant.inspect}"
-              variant.option_values << ov
+              begin
+                # This one line seems to works for 1.1.0 - 3.2 but not 1.0.0 - 3.1 ??
+                if(SpreeHelper::version.to_f >= 1.1)
+                  variant = @load_object.variants.create( :sku => "#{@load_object.sku}_#{i}", :price => @load_object.price, :available_on => @load_object.available_on)
+                else
+                  variant = @@variant_klass.create( :product => @load_object, :sku => "#{@load_object.sku}_#{i}", :price => @load_object.price, :available_on => @load_object.available_on)
+                  #if(variant.valid?)
+                  # variant.save
+                  # @load_object.variants << variant
+               
+                  #else
+                  #puts "WARNING: For Option #{ovname} - Variant creation failed #{variant.errors.inspect}"
+                end
+                variant.option_values << ov
+              rescue =>  e
+                puts "Failed to create a Variant for Product #{@load_object.name}"
+                puts e.inspect
+                puts e.backtrace
+              end
+              
+              logger.debug "Created New Variant: #{variant.inspect}"
+              
             else
               puts "WARNING: Option #{ovname} NOT FOUND - No Variant created"
             end
           end
           
           #puts "DEBUG Load Object now has Variants : #{@load_object.variants.inspect}"
-          @load_object.reload
+          @load_object.reload unless @load_object.new_record? 
           #puts "DEBUG Load Object now has Variants : #{@load_object.variants.inspect}"
         end
 
@@ -212,6 +235,7 @@ module DataShift
           img_path, alt_text = image.split(LoaderBase::name_value_delim)
           
           image = create_image(@@image_klass, img_path, @load_object, :alt => alt_text)
+          logger.debug("Product assigned an Image : #{image.inspect}")
         end
       
       end
@@ -233,10 +257,20 @@ module DataShift
 
           unless property
             property = @@property_klass.create( :name => pname, :presentation => pname.humanize)
+            logger.debug "Created New Property #{property.inspect}"
           end
 
           if(property)
-            @load_object.product_properties << @@product_property_klass.create( :property => property, :value => pvalue)
+            if(SpreeHelper::version.to_f >= 1.1)
+              # Property now protected from mass assignment 
+              x = @@product_property_klass.new( :value => pvalue )
+              x.property = property
+              x.save
+              logger.debug "Created New ProductProperty #{x.inspect}"
+              @load_object.product_properties << x 
+            else
+              @load_object.product_properties << @@product_property_klass.create( :property => property, :value => pvalue)
+            end
           else
             puts "WARNING: Property #{pname} NOT found - Not set Product"
           end
@@ -246,20 +280,6 @@ module DataShift
       end
 
       # Nested tree structure support ..
-      # 
-      # ... inside of main loop
-      # the_taxons = []
-      # taxon_col.split(/[\r\n]+/).each do |chain|
-      #  taxon = nil
-      #   names = chain.split(/\s*>\s*/)
-      #  names.each do |name|
-      #    taxon = Taxon.find_or_create_by_name_and_parent_id_and_taxonomy_id(name, taxon && taxon.id, main_taxonomy.id)
-      #  end
-      #  the_taxons << taxon
-      # end
-      # p.taxons = the_taxons
- 
-      
       # TAXON FORMAT 
       # name|name>child>child|name
        
@@ -302,7 +322,7 @@ module DataShift
           
           unique_list = taxons.compact.uniq - (@load_object.taxons || [])
         
-          #puts "DEBUG: Taxon nms to add #{unique_list.collect(&:name).inspect}"
+          logger.debug("Product assigned to Taxons : #{unique_list.collect(&:name).inspect}")
           @load_object.taxons << unique_list unless(unique_list.empty?)
         end
 

data/lib/thor/export_excel.thor

@@ -0,0 +1,74 @@
+# Copyright:: (c) Autotelik Media Ltd 2012
+# Author ::   Tom Statter
+# Date ::     April 2012
+# License::   MIT.
+#
+# Usage::
+#
+#  To pull Datashift commands into your main application :
+#
+#     require 'datashift'
+#
+#     DataShift::load_commands
+#
+#  Cmd Line:
+#
+# => bundle exec thor datashift:export:excel -m <active record class> -r <output_template.xls> -a
+#
+require 'datashift'
+  
+# Note, not DataShift, case sensitive, create namespace for command line : datashift
+module Datashift
+
+          
+  class Export < Thor     
+  
+    include DataShift::Logging
+      
+    desc "excel", "export any active record model (with optional associations)" 
+    method_option :model, :aliases => '-m', :required => true, :desc => "The active record model to export"
+    method_option :result, :aliases => '-r', :required => true, :desc => "Create template of model in supplied file"
+    method_option :assoc, :aliases => '-a', :type => :boolean, :desc => "Include all associations in the template"
+    method_option :exclude, :aliases => '-e',  :type => :array, :desc => "Use with -a : Exclude association types. Any from #{DataShift::MethodDetail::supported_types_enum.to_a.inspect}"
+    
+    def excel()
+     
+      # TODO - We're assuming run from a rails app/top level dir...
+      # ...can we make this more robust ? e.g what about when using active record but not in Rails app, 
+      require File.expand_path('config/environment.rb')
+   
+      require 'excel_exporter'
+
+      model = options[:model]
+      result = options[:result]
+     
+      logger.info "Datashift: Start Excel export to #{result}"
+            
+      begin
+        # support modules e.g "Spree::Property") 
+        klass = ModelMapper::class_from_string(model)  #Kernel.const_get(model)
+      rescue NameError => e
+        puts e
+        raise "ERROR: No such Model [#{model}] found - check valid model supplied via -model <Class>"
+      end
+
+      begin
+        gen = DataShift::ExcelExporter.new(result)
+
+        if(options[:assoc])
+          opts = (options[:exclude]) ? {:exclude => options[:exclude]} : {}
+          logger.info("Datashift: Exporting with associations")
+          gen.generate_with_associations(klass, opts)
+        else
+          gen.generate(klass)
+        end
+      rescue => e
+        puts e
+        puts e.backtrace
+        puts "Warning: Error during export, data may be incomplete"
+      end
+
+    end
+  end
+
+end

data/lib/thor/spree/products_images.thor

@@ -12,6 +12,7 @@
 
 # Note, not DataShift, case sensitive, create namespace for command line : datashift
 module Datashift
+  
         
   class Spree < Thor     
   
@@ -24,7 +25,7 @@ module Datashift
     method_option :verbose, :aliases => '-v', :type => :boolean, :desc => "Verbose logging"
     method_option :config, :aliases => '-c',  :type => :string, :desc => "Configuration file containg defaults or over rides in YAML"
     
-    def products()  #, [:input, :verbose, :sku_prefix] => :environment do |t, args|
+    def products()
 
       # TODO - We're assuming run from a rails app/top level dir...
       # ...can we make this more robust ? e.g what about when using active record but not in Rails app, 

data/spec/Gemfile

@@ -0,0 +1,18 @@
+source 'https://rubygems.org'
+
+gem 'rspec'  # Behavior Driven Development (BDD) for Ruby
+gem 'rspec-core'  # RSpec runner and example groups.
+gem 'rspec-expectations'  # RSpec matchers for should and should_not.
+gem 'rspec-mocks'  # RSpec test double framework with stubbing and mocking.
+gem 'rspec-rails'  # RSpec version 2.x for Rails version 3.x.
+gem 'activerecord-jdbcsqlite3-adapter'
+
+# DEFINE WHICH VERSIONS WE WANT TO TEST WITH
+
+#gem 'rails', '3.2.3'
+#gem 'spree', '1.1.0'
+
+gem 'rails', '3.1.3'
+gem 'spree', '1.0.0'
+
+

data/spec/excel_exporter_spec.rb

@@ -9,7 +9,7 @@ require File.dirname(__FILE__) + '/spec_helper'
 
 if(Guards::jruby?)
   require 'erb'
-  require 'excel_generator'
+  require 'excel_exporter'
 
   include DataShift
 
@@ -37,37 +37,36 @@ if(Guards::jruby?)
       MethodDictionary.find_operators( @assoc_klazz )
     end
   
-    it "should be able to create a new excel generator" do
+    it "should be able to create a new excel exporter" do
       generator = ExcelExporter.new( 'dummy.xls' )
       
       generator.should_not be_nil
     end
   
-    it "should generate template .xls file from model" do
+    it "should export a model to .xls file" do
 
-      expect = result_file('project_template_spec.xls')
+      expect = result_file('project_export_spec.xls')
 
       gen = ExcelExporter.new( expect )
     
-      gen.generate(Project)
+      gen.export(Project)
  
       File.exists?(expect).should be_true
       
       puts "Can manually check file @ #{expect}"
     end
 
-    it "should export a simple model to .xls spreedsheet" do
+    it "should export a  model and associations to .xls file" do
 
       Project.create( :value_as_string	=> 'Value as Text', :value_as_boolean => true,	:value_as_double => 75.672)
-      #001 Demo string	blah blah	2011-02-14	1.00	320.00
 
-      expect= result_file('simple_export_spec.xls')
+      expect= result_file('project_plus_assoc_export_spec.xls')
 
-      gen = ExcelGenerator.new(expect)
+      gen = ExcelExporter.new(expect)
 
       items = Project.all
 
-      gen.export(items)
+      gen.export_with_associations(Project, items)
 
       File.exists?(expect).should be_true
 

data/spec/excel_generator_spec.rb

@@ -56,18 +56,13 @@ if(Guards::jruby?)
       puts "Can manually check file @ #{expect}"
     end
 
-    it "should export a simple model to .xls spreedsheet" do
+    it "should genrate a complex template .xls file from model" do
 
-      Project.create( :value_as_string	=> 'Value as Text', :value_as_boolean => true,	:value_as_double => 75.672)
-      #001 Demo string	blah blah	2011-02-14	1.00	320.00
-
-      expect= result_file('simple_export_spec.xls')
+      expect= result_file('project_plus_assoc_template_spec.xls')
 
       gen = ExcelGenerator.new(expect)
 
-      items = Project.all
-
-      gen.export(items)
+      gen.generate_with_associations(Project)
 
       File.exists?(expect).should be_true
 

data/spec/spec_helper.rb

@@ -1,5 +1,5 @@
-require 'erb'
-require 'yaml'
+
+require 'stringio'
 
 require File.dirname(__FILE__) + '/../lib/datashift'
 
@@ -14,17 +14,21 @@ include DataShift
 #
 #
 # We are not setup as a Rails project so need to mimic an active record database setup so
-# we have some  AR models top test against. Create an in memory database from scratch.
+# we have some  AR models to test against. Create an in memory database from scratch.
 #
 $DataShiftFixturePath = File.join(File.dirname(__FILE__), 'fixtures')
 $DataShiftDatabaseYml = File.join($DataShiftFixturePath, 'config/database.yml')
 
 module DataShift
  
-  def db_connect( env = 'test_file', version = nil)
+  def db_connect( env = 'test_file')
+        
+    gemfile =  File.join(DataShift::root_path, 'spec', 'Gemfile')
+
+    ENV['BUNDLE_GEMFILE'] = gemfile
+    require 'bundler'
+    Bundler.setup
 
-    version ? gem('activerecord', version) : gem('activerecord')
-    
     require 'active_record'
 
     # Some active record stuff seems to rely on the RAILS_ENV being set ?
@@ -41,13 +45,13 @@ module DataShift
     
     #dbtype = Rails.configuration.database_configuration[Rails.env]['adapter'].to_sym
 
-    #ActiveRecord::Base.logger = Logger.new(STDOUT)
-
     require 'logger'
     logdir = File.dirname(__FILE__) + '/logs'
     FileUtils.mkdir_p(logdir) unless File.exists?(logdir)
     ActiveRecord::Base.logger = Logger.new(logdir + '/datashift_spec.log')
 
+    # Anyway to direct one logger to another ????? ... Logger.new(STDOUT)
+    
     @dslog = ActiveRecord::Base.logger
 
     puts "Connecting to DB"
@@ -57,6 +61,8 @@ module DataShift
     # so copied this from ... Rails::Initializer.initialize_cache
     Object.const_set "RAILS_CACHE", ActiveSupport::Cache.lookup_store( :memory_store )
 
+    puts "Connected to DB"
+    
     @dslog.info "Connected to DB - #{ActiveRecord::Base.connection.inspect}"
   end
 
@@ -100,6 +106,8 @@ end
 
 module SpecHelper
   
+  # VERSIONS of Spree (1.1.0.rc1, 1.0.0, 0.11.2)
+  
   $SpreeFixturePath = File.join($DataShiftFixturePath, 'spree')    
   $SpreeNegativeFixturePath = File.join($DataShiftFixturePath, 'negative')
     
@@ -108,11 +116,14 @@ module SpecHelper
   end
   
   def self.before_all_spree 
+
     # we are not a Spree project, nor is it practical to externally generate
     # a complete Spree application for testing so we implement a mini migrate/boot of our own
-    SpreeHelper.load()          # require Spree gems
-    SpreeHelper.boot( 'test_spree_standalone' )             # key to YAML db e.g  test_memory, test_mysql
+    SpreeHelper.load()        
+    SpreeHelper.boot('test_spree_standalone')             # key to YAML db e.g  test_memory, test_mysql
     
+    puts "Testing Spree standalone - version #{SpreeHelper::version}"
+        
     SpreeHelper.migrate_up      # create an sqlite Spree database on the fly
   end
   
@@ -136,6 +147,19 @@ module SpecHelper
       
 end
 
+def capture(stream)
+  begin
+    stream = stream.to_s
+    eval "$#{stream} = StringIO.new"
+    yield
+    result = eval("$#{stream}").string
+  ensure
+    eval("$#{stream} = #{stream.upcase}")
+  end
+
+  result
+end
+  
 RSpec.configure do |config|
   # config.use_transactional_fixtures = true
   # config.use_instantiated_fixtures  = false

data/spec/spree_exporter_spec.rb

@@ -12,7 +12,7 @@
 require File.dirname(__FILE__) + '/spec_helper'
 
 require 'spree_helper'
-require 'excel_generator'
+require 'excel_exporter'
 
 include DataShift
   
@@ -20,6 +20,7 @@ describe 'SpreeLoader' do
 
   before(:all) do
     SpecHelper::before_all_spree
+    results_clear()
   end
 
   before do
@@ -29,36 +30,38 @@ describe 'SpreeLoader' do
       
     before_each_spree   # inits tests, cleans DB setups model types
     
+    @Taxon_klass.delete_all
+    
     # Create some test data
     root = @Taxonomy_klass.create( :name => 'Paintings' )
     
-    @Taxon_klass.create( :name => 'Landscape', :taxonomy => root )
-    @Taxon_klass.create( :name => 'Sea', :taxonomy => root )
+    @Taxon_klass.create( :name => 'Landscape', :description => "Nice paintings", :taxonomy_id => root.id )
+    @Taxon_klass.create( :name => 'Sea', :description => "Waves and sand", :taxonomy_id => root.id )
       
   end
 
   it "should export any Spree model to .xls spreedsheet" do
 
-    expect = result_file('taxonomy_export_spec.xls')
+    expect = result_file('taxon_export_spec.xls')
 
-    gen = ExcelGenerator.new(expect)
+    exporter = ExcelExporter.new(expect)
 
-    items = @Taxonomy_klass.all
+    items = @Taxon_klass.all
 
-    gen.export(items)
+    exporter.export(items)
 
     File.exists?(expect).should be_true
   end
 
   it "should export a Spree model and associations to .xls spreedsheet" do
 
-    expect = result_file('taxonomy_and_assoc_export_spec.xls')
+    expect = result_file('taxon_and_assoc_export_spec.xls')
 
-    gen = ExcelGenerator.new(expect)
+    exporter = ExcelExporter.new(expect)
 
-    items = @Taxonomy_klass.all
+    items = @Taxon_klass.all
       
-    gen.generate_with_associations(@Taxonomy_klass, items)
+    exporter.export_with_associations(@Taxon_klass, items)
 
     File.exists?(expect).should be_true
 

data/spec/spree_generator_spec.rb

@@ -24,7 +24,7 @@ describe 'SpreeLoader' do
 
   before do
     
-    include SpecHelper
+    #include SpecHelper
     extend SpecHelper
       
     before_each_spree   # inits tests, cleans DB setups model types

data/spec/spree_images_loader_spec.rb

@@ -45,29 +45,45 @@ describe 'SpreeLoader' do
   end
 
 
-  it "should load Products with associated image" do
+  it "should load Products with associated image from CSV" do
+    
+    # In >= 1.1.0 Image moved to master Variant from Product
+    if(SpreeHelper::version.to_f < 1.1)
    
-    @product_loader.perform_load( SpecHelper::spree_fixture('SpreeProductsWithImages.csv'), :mandatory => ['sku', 'name', 'price'] )
+      @product_loader.perform_load( SpecHelper::spree_fixture('SpreeProductsWithImages.csv'), :mandatory => ['sku', 'name', 'price'] )
      
-    p = @klass.find_by_name("Demo Product for AR Loader")
+      p = @klass.find_by_name("Demo Product for AR Loader")
     
-    p.name.should == "Demo Product for AR Loader"
-    p.images.should have_exactly(1).items
+      p.name.should == "Demo Product for AR Loader"
+      p.images.should have_exactly(1).items
     
-    @klass.all.each {|p| p.images.should have_exactly(1).items }
+      @klass.all.each {|p| p.images.should have_exactly(1).items }
+    end
   end
   
   
-  it "should be able to assign Images to preloaded Products" do
+  it "should load Products with associated image", :fail => true do
+   
+    if(SpreeHelper::version.to_f < 1.1)
+      @product_loader.perform_load( SpecHelper::spree_fixture('SpreeProductsWithImages.xls'), :mandatory => ['sku', 'name', 'price'] )
+     
+      p = @klass.find_by_name("Demo Product for AR Loader")
     
-    @product_loader.perform_load( SpecHelper::spree_fixture('SpreeProductsMultiColumn.csv'), :mandatory => ['sku', 'name', 'price'] )
-  
-    @Image_klass.all.size.should == 0
+      p.name.should == "Demo Product for AR Loader"
+      p.images.should have_exactly(1).items
     
-    @klass.all.each {|p| p.images.should have_exactly(0).items }
+      @klass.all.each {|p| p.images.should have_exactly(1).items }
+    end
+  end
+  
+  it "should be able to assign Images to preloaded Products" do
     
+    @Image_klass.all.size.should == 0
+
     loader = DataShift::SpreeHelper::ImageLoader.new
     
+    loader.perform_load( SpecHelper::spree_fixture('SpreeProductImages.xls') )
+    
   end
   
 end

data/spec/spree_loader_spec.rb

@@ -32,17 +32,19 @@ describe 'SpreeLoader' do
       
       before_each_spree
     
-      @klass.count.should == 0
+      @Product_klass.count.should == 0
       @Taxon_klass.count.should == 0
       @Variant_klass.count.should == 0
       
       MethodDictionary.clear
-      MethodDictionary.find_operators( @klass )
+      MethodDictionary.find_operators( @Product_klass )
 
       # want to test both lookup and dynamic creation - this Taxonomy should be found, rest created
       root = @Taxonomy_klass.create( :name => 'Paintings' )
     
-      @Taxon_klass.create( :name => 'Landscape', :taxonomy => root )
+      t = @Taxon_klass.new( :name => 'Landscape' )
+      t.taxonomy = root
+      t.save
 
       @Taxon_klass.count.should == 2
     
@@ -99,14 +101,14 @@ describe 'SpreeLoader' do
     
     @product_loader.perform_load( SpecHelper::spree_fixture(source), :mandatory => ['sku', 'name', 'price'] )
 
-    @klass.count.should == 3
+    @Product_klass.count.should == 3
 
     @product_loader.failed_objects.size.should == 0
     @product_loader.loaded_objects.size.should == 3
 
-    @product_loader.loaded_count.should == @klass.count
+    @product_loader.loaded_count.should == @Product_klass.count
 
-    p = @klass.first
+    p = @Product_klass.first
     
     p.sku.should == "SIMPLE_001"
     p.price.should == 345.78
@@ -116,7 +118,7 @@ describe 'SpreeLoader' do
     p.option_types.should have_exactly(1).items
     p.option_types.should have_exactly(1).items
     p.count_on_hand.should == 12
-    @klass.last.count_on_hand.should == 23
+    @Product_klass.last.count_on_hand.should == 23
   end
 
   
@@ -149,16 +151,16 @@ describe 'SpreeLoader' do
   def test_default_values
     @product_loader.perform_load( SpecHelper::spree_fixture('SpreeProductsMandatoryOnly.xls'), :mandatory => ['sku', 'name', 'price'] )
     
-    @klass.count.should == 3
+    @Product_klass.count.should == 3
 
     @product_loader.failed_objects.size.should == 0
     @product_loader.loaded_objects.size.should == 3
     
-    p = @klass.first
+    p = @Product_klass.first
     
     p.sku.should == "SPEC_SIMPLE_001"
       
-    @klass.all { |p|
+    @Product_klass.all { |p|
       p.sku.should.include "SPEC_"
       p.cost_price = 1.0
       p.available_on.should == @expected_time
@@ -180,7 +182,7 @@ describe 'SpreeLoader' do
   end
   
   def test_variants_creation( source )
-    @klass.count.should == 0
+    @Product_klass.count.should == 0
     @Variant_klass.count.should == 0
     
     @product_loader.perform_load( SpecHelper::spree_fixture(source), :mandatory => ['sku', 'name', 'price'] )
@@ -192,10 +194,10 @@ describe 'SpreeLoader' do
   def expected_multi_column_variants
       
     # 3 MASTER products, 11 VARIANTS
-    @klass.count.should == 3
+    @Product_klass.count.should == 3
     @Variant_klass.count.should == 14
 
-    p = @klass.first
+    p = @Product_klass.first
 
     p.sku.should == "DEMO_001"
 
@@ -204,7 +206,7 @@ describe 'SpreeLoader' do
     p.description.should == "blah blah"
     p.cost_price.should == 320.00
 
-    @klass.all.select {|m| m.is_master.should == true  }
+    @Product_klass.all.select {|m| m.is_master.should == true  }
 
 
     # mime_type:jpeg mime_type:PDF mime_type:PNG
@@ -281,12 +283,12 @@ describe 'SpreeLoader' do
   
   def expected_multi_column_properties
     # 3 MASTER products, 11 VARIANTS
-    @klass.count.should == 3
+    @Product_klass.count.should == 3
     @Variant_klass.count.should == 14
 
-    @klass.first.properties.should have_exactly(1).items
+    @Product_klass.first.properties.should have_exactly(1).items
 
-    p3 = @klass.all.last
+    p3 = @Product_klass.all.last
 
     p3.properties.should have_exactly(3).items
 
@@ -304,7 +306,7 @@ describe 'SpreeLoader' do
   # Operation and results should be identical when loading multiple associations
   # if using either single column embedded syntax, or one column per entry.
 
-  it "should load Products and multiple Taxons from single column", :taxon => true do
+  it "should load Products and multiple Taxons from single column", :taxons => true do
     test_taxon_creation( 'SpreeProducts.xls' )
   end
 
@@ -329,7 +331,7 @@ describe 'SpreeLoader' do
     #puts @Taxonomy_klass.all.collect( &:name).inspect
     #puts @Taxon_klass.all.collect( &:name).inspect
     
-    # Paintings alreadyexisted and had 1 child Taxon (Landscape)
+    # Paintings already existed and had 1 child Taxon (Landscape)
     # 2 nested Taxon (Paintings>Nature>Seascape) created under it so expect Taxonomy :
     
     # WaterColour	
@@ -340,13 +342,16 @@ describe 'SpreeLoader' do
     @Taxonomy_klass.count.should == 4
     @Taxon_klass.count.should == 7
 
-    @klass.first.taxons.should have_exactly(2).items
-    @klass.last.taxons.should have_exactly(2).items
+    @Product_klass.first.taxons.should have_exactly(2).items
+    @Product_klass.last.taxons.should have_exactly(2).items
 
     p2 = @Variant_klass.find_by_sku("DEMO_002").product
 
     # Paintings	Oils	Paintings>Nature>Seascape
 
+    # ["Nature", "Paintings", "Seascape"]
+
+    
     #puts p2.taxons.collect(&:name).inspect
       
     p2.taxons.should have_exactly(4).items
@@ -354,7 +359,10 @@ describe 'SpreeLoader' do
     p2.taxons.collect(&:name).sort.should == ['Nature','Oils','Paintings','Seascape']
      
     paint_parent = @Taxonomy_klass.find_by_name('Paintings')
-         
+    
+       
+    puts paint_parent.taxons.collect(&:name).sort.inspect
+     
     paint_parent.taxons.should have_exactly(4).items # 3 children + all Taxonomies have a root Taxon
     
     paint_parent.taxons.collect(&:name).sort.should == ['Landscape','Nature','Paintings','Seascape']
@@ -367,16 +375,13 @@ describe 'SpreeLoader' do
     
     p2.taxons.collect( &:id ).should include(ts.id)
     p2.taxons.collect( &:id ).should include(tn.id)
-    
-    puts tn.inspect
-    puts ts.inspect
+
      
     tn.parent.id.should == paint_parent.root.id
     ts.parent.id.should == tn.id
     
     tn.children.should have_exactly(1).items
     ts.children.should have_exactly(0).items
-
  
   end
   
@@ -392,7 +397,7 @@ describe 'SpreeLoader' do
   end
   
   
-  it "should load Products from multiple column csv as per .xls" do
+  it "should load Products from multiple column csv as per .xls", :blah => true do
     test_variants_creation('SpreeProductsMultiColumn.csv')
     
     expected_multi_column_properties

data/spec/thor_spec.rb

@@ -0,0 +1,32 @@
+# Copyright:: (c) Autotelik Media Ltd 2012
+# Author ::   Tom Statter
+# Date ::     April 20121
+#
+# License::   MIT - Free, OpenSource
+#
+# Details::   Specification for Thor tasks supplied with datashift
+#
+require 'thor'
+require 'thor/group'
+
+
+require File.dirname(__FILE__) + '/spec_helper'
+
+require 'spree_helper'
+
+load 'products_images.thor'
+
+include DataShift
+  
+describe 'Thor high level command line tasks' do
+
+  before(:each) do
+  end
+
+  
+  it "should be able to run spree loaders from a simple command line task" do
+    Datashift::Spree.start(["products"])
+  end
+
+    
+end

metadata

@@ -2,7 +2,7 @@
 name: datashift
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.4.3
+  version: 0.5.0
 platform: ruby
 authors: 
   - Thomas Statter
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-03-31 00:00:00 Z
+date: 2012-05-08 00:00:00 Z
 dependencies: []
 
 description: A suite of tools to move data between ActiveRecord models,databases,applications like Excel/Open Office, files and projects including Spree
@@ -30,6 +30,7 @@ files:
   - README.rdoc
   - Rakefile
   - VERSION
+  - datashift-0.5.0.gem
   - datashift.gemspec
   - lib/applications/jruby/jexcel_file.rb
   - lib/applications/jruby/word.rb
@@ -74,6 +75,7 @@ files:
   - lib/loaders/spreadsheet_loader.rb
   - lib/loaders/spree/image_loader.rb
   - lib/loaders/spree/product_loader.rb
+  - lib/thor/export_excel.thor
   - lib/thor/generate_excel.thor
   - lib/thor/import_excel.thor
   - lib/thor/spree/bootstrap_cleanup.thor
@@ -98,6 +100,7 @@ files:
   - sandbox/config/database.yml
   - sandbox/config/environment.rb
   - sandbox/config/environments/development.rb
+  - spec/Gemfile
   - spec/csv_loader_spec.rb
   - spec/datashift_spec.rb
   - spec/db/migrate/20110803201325_create_test_bed.rb
@@ -119,13 +122,13 @@ files:
   - spec/fixtures/images/DEMO_002_Powerstation.jpg
   - spec/fixtures/images/DEMO_003_ror_mug.jpeg
   - spec/fixtures/images/DEMO_004_ror_ringer.jpeg
-  - spec/fixtures/interact_models_db.sqlite
   - spec/fixtures/negative/SpreeProdMiss1Mandatory.csv
   - spec/fixtures/negative/SpreeProdMiss1Mandatory.xls
   - spec/fixtures/negative/SpreeProdMissManyMandatory.csv
   - spec/fixtures/negative/SpreeProdMissManyMandatory.xls
   - spec/fixtures/simple_export_spec.xls
   - spec/fixtures/simple_template_spec.xls
+  - spec/fixtures/spree/SpreeProductImages.xls
   - spec/fixtures/spree/SpreeProducts.csv
   - spec/fixtures/spree/SpreeProducts.xls
   - spec/fixtures/spree/SpreeProductsDefaults.yml
@@ -148,6 +151,7 @@ files:
   - spec/spree_images_loader_spec.rb
   - spec/spree_loader_spec.rb
   - spec/spree_method_mapping_spec.rb
+  - spec/thor_spec.rb
   - tasks/config/seed_fu_product_template.erb
   - tasks/config/tidy_config.txt
   - tasks/export/excel_generator.rake