datashift 0.5.0 → 0.6.0

data/lib/loaders/spree/image_loader.rb

@@ -12,35 +12,48 @@ module DataShift
  
     include DataShift::Logging
      
+    def get_file( attachment_path )
+      
+      unless File.exists?(attachment_path) && File.readable?(attachment_path)
+        logger.error("Cannot process Image from #{Dir.pwd}: Invalid Path #{attachment_path}")
+        raise "Cannot process Image : Invalid Path #{attachment_path}"
+      end
+     
+      file = begin
+        File.new(attachment_path, "rb")
+      rescue => e
+        puts e.inspect
+        raise "ERROR : Failed to read image #{attachment_path}"
+      end
+      
+      file
+    end
+    
     # Note the Spree Image model sets default storage path to
     # => :path => ":rails_root/public/assets/products/:id/:style/:basename.:extension"
 
-    def create_image(klass, image_path, viewable_record = nil, options = {})
+    def create_image(klass, attachment_path, viewable_record = nil, options = {})
        
-      unless File.exists?(image_path) && File.readable?(image_path)
-        logger.error("Cannot process Image from #{Dir.pwd}: Invalid Path #{image_path}")
-        raise "Cannot process Image : Invalid Path #{image_path}"
-      end
-      
       alt = if(options[:alt])
         options[:alt]
       else
         (viewable_record and viewable_record.respond_to? :name) ? viewable_record.name : ""
       end
-     
-      file = begin
-        File.new(image_path, "rb")
-      rescue => e
-        puts e.inspect
-        raise "ERROR : Failed to read image #{image_path}"
-      end
-
+    
       position = (viewable_record and viewable_record.respond_to?(:images)) ? viewable_record.images.length : 0
+          
+      file = get_file(attachment_path)
       
-      image = klass.new( :attachment => file,:viewable => viewable_record, :alt => alt, :position => position)  
+      if(SpreeHelper::version.to_f > 1 && viewable_record.is_a?(Spree::Product) )
+       
+        image = klass.new( :attachment => file, :alt => alt, :position => position)  
+        
+        # mass assignment not allows for this field
+        image.viewable = viewable_record.master
+      else
+        image = klass.new( :attachment => file,:viewable => viewable_record, :alt => alt, :position => position)  
+      end
       #image.attachment.reprocess!
-   
-      #image.viewable =  viewable_record if viewable_record
 
       puts image.save ? "Success: Created Image: #{image.inspect}" : "ERROR : Problem saving to DB Image: #{image.inspect}"
     end
@@ -55,18 +68,53 @@ module DataShift
       include DataShift::ExcelLoading
       
       def initialize(image = nil)
-        puts SpreeHelper::get_spree_class('Image')
+        super( SpreeHelper::get_spree_class('Image'), image )
         
-        @@image_klass ||= SpreeHelper::get_spree_class('Image')
-            
-        super( @@image_klass, image )
         raise "Failed to create Image for loading" unless @load_object
       end
 
-      # The path to the physical image on local disk
-      def process(image_path, record = nil)
-        @load_object = create_image(@@image_klass, image_path, record)
+      def sku_klazz
+        @sku_klazz ||= SpreeHelper::get_spree_class('Variant' )
+        @sku_klazz
+      end
+      
+      def process()
+
+        if(current_value && @current_method_detail.operator?('attachment') )
+          @load_object.attachment = get_file(current_value)
+          
+          puts "Image attachment set : #{@load_object.inspect}"
+              
+        elsif(current_value && @current_method_detail.operator?('sku') )    
+          
+          return if(current_value.empty?)
+          puts "Looking for record with SKU #{current_value}"  
+          add_record( sku_klazz.find_by_sku(current_value) )
+               
+        elsif(current_value && @current_method_detail.operator?('name') )   
+                     
+          puts "Looking for record with NAME [#{current_value}]"
+          add_record attachment_klazz.find_by_name(current_value)
+        
+        end
+          
+      end
+      
+      def add_record(record)
+        if(record)
+          if(SpreeHelper::version.to_f > 1 )
+            @load_object.viewable = record
+          else
+            @load_object.viewable = record.product   # SKU stored on Variant but we want it's master Product
+          end
+          @load_object.save
+          puts "Image viewable set : #{record.inspect}"
+          
+        else
+          logger.error"Failed to find a matching record"
+        end
       end
     end
+          
   end
 end

data/lib/loaders/spree/product_loader.rb

@@ -39,30 +39,7 @@ module DataShift
         logger.debug "PRODUCT #{@load_object.inspect} MASTER: #{@load_object.master.inspect}"
       end
 
-      # Based on filename call appropriate loading function
-      # Currently supports :
-      #   Excel/Open Office files saved as .xls
-      #   CSV files
-      # 
-      # OPTIONS :
-      #   strict : Raise exception if any column cannot be mapped
-       
-      def perform_load( file_name, options = {} )
 
-        raise DataShift::BadFile, "Cannot load #{file_name} file not found." unless(File.exists?(file_name))
-        
-        ext = File.extname(file_name)
-          
-        if(ext.casecmp('.xls') == 0)
-          raise DataShift::BadRuby, "Please install and use JRuby for loading .xls files" unless(Guards::jruby?)
-          perform_excel_load(file_name, options)
-        elsif(ext.casecmp('.csv') == 0)
-          perform_csv_load(file_name, options)
-        else
-          raise DataShift::UnsupportedFileType, "#{ext} files not supported - Try .csv or OpenOffice/Excel .xls"
-        end
-      end
-      
       # Over ride base class process with some Spree::Product specifics
       #
       # What process a value string from a column, assigning value(s) to correct association on Product.
@@ -138,12 +115,7 @@ module DataShift
 
       private
       
-      # Take current column data and split into each association
-      # Supported Syntax :
-      #  assoc_find_name:value | assoc2_find_name:value | etc
-      def get_each_assoc
-        current_value.to_s.split( LoaderBase::multi_assoc_delim )
-      end
+
 
       # Special case for OptionTypes as it's two stage process
       # First add the possible option_types to Product, then we are able
@@ -189,7 +161,7 @@ module DataShift
               begin
                 # This one line seems to works for 1.1.0 - 3.2 but not 1.0.0 - 3.1 ??
                 if(SpreeHelper::version.to_f >= 1.1)
-                  variant = @load_object.variants.create( :sku => "#{@load_object.sku}_#{i}", :price => @load_object.price, :available_on => @load_object.available_on)
+                  variant = @load_object.variants.create( :sku => "#{@load_object.sku}_#{i}", :price => @load_object.price)
                 else
                   variant = @@variant_klass.create( :product => @load_object, :sku => "#{@load_object.sku}_#{i}", :price => @load_object.price, :available_on => @load_object.available_on)
                   #if(variant.valid?)
@@ -203,7 +175,7 @@ module DataShift
               rescue =>  e
                 puts "Failed to create a Variant for Product #{@load_object.name}"
                 puts e.inspect
-                puts e.backtrace
+                #puts e.backtrace
               end
               
               logger.debug "Created New Variant: #{variant.inspect}"

data/lib/thor/import_excel.thor

@@ -11,15 +11,11 @@
 #
 #     DataShift::load_commands
 #
-#  Requires Jruby, cmd Line:
+#  N.B Requires JRuby
 #
-# => bundle exec thor datashift:import:excel -m <active record class> -r <output_template.xls> -a
-#
-#  Cmd Line:
-#
-# => jruby -S rake datashift:import:excel model=<active record class> input=<file.xls>
-# => jruby -S rake datashift:import:excel model=<active record class> input=C:\MyProducts.xlsverbose=true
+# => bundle exec thor datashift:import:excel -m <active record class> -i <output_template.xls> -a
 #
+
 require 'datashift'
   
 
@@ -34,6 +30,7 @@ module Datashift
     desc "excel", "import .xls file for specifiec active record model" 
     method_option :model, :aliases => '-m', :required => true, :desc => "The related active record model"
     method_option :input, :aliases => '-i', :required => true, :desc => "The input .xls file"
+    method_option :config, :aliases => '-c', :desc => "YAML config file with defaults, over-rides etc"
     method_option :assoc, :aliases => '-a', :type => :boolean, :desc => "Include any associations supplied in the input"
     method_option :exclude, :aliases => '-e',  :type => :array, :desc => "Use with -a : Exclude association types. Any from #{DataShift::MethodDetail::supported_types_enum.to_a.inspect}"
     
@@ -48,16 +45,17 @@ module Datashift
       model = options[:model]
       begin
         # support modules e.g "Spree::Property") 
-        klass = ModelMapper::class_from_string(model)  #Kernel.const_get(model)
+        klass = ModelMapper::class_from_string(model) 
       rescue NameError
-        raise "ERROR: No such AR Model found - check valid model supplied via model=<Class>"
+        raise "ERROR: No such AR Model found - check valid model supplied with -m <Class>"
       end
 
-      if(ENV['loader'])
+      raise "ERROR: No such AR Model found - check valid model supplied with -m <Class>" if(klass.nil?) 
+      
+      if(options[:loader])
         begin
-          #loader_klass = Kernel.const_get(ENV['loader'])
-          # support modules e.g "Spree::Property") 
-          loader_klass = ModelMapper::class_from_string(ENV['loader'])  #Kernel.const_get(model)
+     
+          loader_klass = ModelMapper::class_from_string(options[:loader])  
 
           loader = loader_klass.new(klass)
 
@@ -71,11 +69,12 @@ module Datashift
         loader = DataShift::ExcelLoader.new(klass)
       end
 
-      logger.info("ARGS #{options.inspect} [#{options[:verbose]}]")
+      logger.info("ARGS #{options.inspect}")
       loader.logger.verbose if(ENV['verbose'])
       
-      loader.configure_from( ENV['config'] ) if(ENV['config'])
-       
+      loader.configure_from( options[:config] ) if(options[:config])
+
+
       loader.perform_load(options[:input])
     end
   end

data/lib/thor/spree/products_images.thor

@@ -77,7 +77,7 @@ module Datashift
     method_option :verbose, :aliases => '-v', :type => :boolean, :desc => "Verbose logging"
     #method_option :config, :aliases => '-c',  :type => :string, :desc => "Configuration file containg defaults or over rides in YAML"
    
-    def images()#, [:input, :folder, :dummy, :sku, :skip_if_no_assoc, :skip_if_loaded, :model] => :environment do |t, args|
+    def images()
 
       require File.expand_path('config/environment.rb')
       
@@ -91,6 +91,7 @@ module Datashift
      
       attachment_klazz  = DataShift::SpreeHelper::get_spree_class('Product' )
       sku_klazz         = DataShift::SpreeHelper::get_spree_class('Variant' )
+      image_klazz       = DataShift::SpreeHelper::get_spree_class('Image' )
 
       # TODO generalise for any paperclip project, for now just Spree
       #begin
@@ -152,8 +153,8 @@ module Datashift
           # Check if Image must have an associated record
           if(record || (record.nil? && options[:process_when_no_assoc]))
             image_loader.reset()
-            puts "Process Image"
-            image_loader.process( image_name, record )
+            puts "Processing Image #{image_name}"
+            image_loader.create_image(image_klazz, image_name, record)
           end
 
         end
@@ -168,6 +169,7 @@ module Datashift
           end
         end
 
+        puts "Dummy Run - if happy run without -d" if(options[:dummy])
       else
         puts "ERROR: Supplied Path #{@image_cache} not accesible"
         exit(-1)

data/sandbox/config/application.rb

@@ -1,11 +1,16 @@
+require File.expand_path('../boot', __FILE__)
+
 require 'rails/all'
 
-module SpreeHelper
-  class Application < Rails::Application
-  
-    config.to_prepare do
-    end
+if defined?(Bundler)
+  # If you precompile assets before deploying to production, use this line
+  Bundler.require(*Rails.groups(:assets => %w(development test)))
+  # If you want your assets lazily compiled in production, use this line
+  # Bundler.require(:default, :assets, Rails.env)
+end
 
+module Sandbox
+  class Application < Rails::Application
     # Settings in config/environments/* take precedence over those specified here.
     # Application configuration should go into files in config/initializers
     # -- all .rb files in that directory are automatically loaded.
@@ -34,6 +39,17 @@ module SpreeHelper
     # Configure sensitive parameters which will be filtered from the log file.
     config.filter_parameters += [:password]
 
+    # Use SQL instead of Active Record's schema dumper when creating the database.
+    # This is necessary if your schema can't be completely dumped by the schema dumper,
+    # like if you have constraints or database-specific column types
+    # config.active_record.schema_format = :sql
+
+    # Enforce whitelist mode for mass assignment.
+    # This will create an empty whitelist of attributes available for mass-assignment for all models
+    # in your app. As such, your models will need to explicitly whitelist or blacklist accessible
+    # parameters by using an attr_accessible or attr_protected declaration.
+    config.active_record.whitelist_attributes = true
+
     # Enable the asset pipeline
     config.assets.enabled = true
 

data/sandbox/config/database.yml

@@ -1,34 +1,20 @@
-# Used by spec helper to build test databases
-# These entires can be used in call to db_connect ( see spec_helper )
-
-<% adapter = 'sqlite3' %>
-<% adapter = 'jdbcsqlite3' if(Guards::jruby? ) %>
-
-test_mysql:
-  adapter: jdbcmysql
-  database: datashift_test_models_db
-  username: test
-  password: test
-  host: localhost
-
-test_memory:
-  adapter: <%= adapter %>
-  database: :memory
+# SQLite version 3.x
+#   gem 'activerecord-jdbcsqlite3-adapter'
+#
+# Configure Using Gemfile
+# gem 'activerecord-jdbcsqlite3-adapter'
+#
+development:
+  adapter: sqlite3
+  database: db/development.sqlite3
 
-test_file:
-  adapter: <%= adapter %>
-  database: <%= File.join($DataShiftFixturePath, 'datashift_test_models_db.sqlite') %>
-  encoding: utf8
-  host: localhost
+# Warning: The database defined as "test" will be erased and
+# re-generated from your development database when you run "rake".
+# Do not set this db to the same as development or production.
+test:
+  adapter: sqlite3
+  database: db/test.sqlite3
 
-test_spree_standalone:
-  adapter: <%= adapter %>
-  database: <%= File.join($DataShiftFixturePath, 'datashift_Spree_db.sqlite') %>
-  encoding: utf8
-  host: localhost
-  
-development:
-  adapter: <%= adapter %>
-  database: <%= File.join($DataShiftFixturePath, 'datashift_Spree_db.sqlite') %>
-  encoding: utf8
-  host: localhost
+production:
+  adapter: sqlite3
+  database: db/production.sqlite3

data/sandbox/config/environment.rb

@@ -2,6 +2,4 @@
 require File.expand_path('../application', __FILE__)
 
 # Initialize the rails application
-SpreeHelper::Application.initialize!
-
-ActiveRecord::Base.include_root_in_json = true
+Sandbox::Application.initialize!

data/sandbox/config/environments/development.rb

@@ -1,8 +1,8 @@
-SpreeHelper::Application.configure do
+Sandbox::Application.configure do
   # Settings specified here will take precedence over those in config/application.rb
 
   # In the development environment your application's code is reloaded on
-  # every request.  This slows down response time but is perfect for development
+  # every request. This slows down response time but is perfect for development
   # since you don't have to restart the web server when you make code changes.
   config.cache_classes = false
 
@@ -22,6 +22,13 @@ SpreeHelper::Application.configure do
   # Only use best-standards-support built into browsers
   config.action_dispatch.best_standards_support = :builtin
 
+  # Raise exception on mass assignment protection for Active Record models
+  config.active_record.mass_assignment_sanitizer = :strict
+
+  # Log the query plan for queries taking more than this (works
+  # with SQLite, MySQL, and PostgreSQL)
+  config.active_record.auto_explain_threshold_in_seconds = 0.5
+
   # Do not compress assets
   config.assets.compress = false
 

data/spec/Gemfile

@@ -5,14 +5,25 @@ gem 'rspec-core'  # RSpec runner and example groups.
 gem 'rspec-expectations'  # RSpec matchers for should and should_not.
 gem 'rspec-mocks'  # RSpec test double framework with stubbing and mocking.
 gem 'rspec-rails'  # RSpec version 2.x for Rails version 3.x.
-gem 'activerecord-jdbcsqlite3-adapter'
 
+# we need both, for JRuby testing of Excel and non JRuby csv
+platform :jruby do
+  gem 'jruby-openssl'
+  gem 'activerecord-jdbcsqlite3-adapter'
+end
+
+platform :ruby do 
+  gem 'sqlite3'
+end
+       
 # DEFINE WHICH VERSIONS WE WANT TO TEST WITH
 
-#gem 'rails', '3.2.3'
-#gem 'spree', '1.1.0'
+gem 'spreadsheet'
+
+gem 'rails', '3.2.3'
+gem 'spree', '1.1.1'
 
-gem 'rails', '3.1.3'
-gem 'spree', '1.0.0'
+#gem 'rails', '3.1.3'
+#gem 'spree', '1.0.0'