datashift 0.15.0 → 0.16.0

data/lib/loaders/paperclip/attachment_loader.rb

@@ -50,11 +50,11 @@ module DataShift
       #         :attach_to_field => avatar    : User.avatar = attachment
       #         
       #       
-      def initialize(attachment_klazz, find_operators = true, attachment = nil, options = {})
+      def initialize(attachment_klazz, attachment = nil, options = {})
         
         init_from_options( options )
  
-        super( attachment_klazz, find_operators, attachment, options.dup )
+        super( attachment_klazz, attachment, options.dup )
          
         puts "Attachment Class is #{load_object_class}" if(@verbose)
        
@@ -153,8 +153,8 @@ module DataShift
           # Check if attachment must have an associated record
           if(record)
             reset()
-          
-            create_attachment(@load_object_class, file_name, record, attach_to_field, options)
+
+            create_paperclip_attachment(@load_object_class, file_name, record, attach_to_field, options)
    
             puts "Added Attachment #{File.basename(file_name)} to #{record.send(attach_to_find_by_field)}(id : #{record.id})" if(@verbose)
           end

data/lib/loaders/paperclip/datashift_paperclip.rb

@@ -72,7 +72,9 @@ module DataShift
     #
     #       Give : { :has_attached_file_attribute => :icon }
     #
-    def create_attachment(klass, attachment_path, record = nil, attach_to_record_field = nil, options = {})
+    def create_paperclip_attachment(klass, attachment_path, record = nil, attach_to_record_field = nil, options = {})
+
+      logger.info("Paperclip::create_paperclip_attachment on Class #{klass}")
 
       has_attached_file_attribute = options[:has_attached_file_name] ? options[:has_attached_file_name].to_sym : :attachment
 
@@ -87,7 +89,7 @@ module DataShift
         @attachment = klass.new(paperclip_attributes, :without_protection => true)
       rescue => e
         logger.error( e.backtrace)
-        logger.error("Failed to create PaperClip Attachment : #{e.inspect}")
+        logger.error("Failed to create PaperClip Attachment for cl;ass #{klass} : #{e.inspect}")
         raise CreateAttachmentFailed.new("Failed to create PaperClip Attachment from : #{attachment_path}")
       ensure
         attachment_file.close unless attachment_file.closed?
@@ -99,7 +101,7 @@ module DataShift
           puts "Success: Created Attachment #{@attachment.id} : #{@attachment.attachment_file_name}"
 
           if(attach_to_record_field.is_a? MethodDetail)
-            DataShift::Populator.new().assign(attach_to_record_field, record, @attachment)
+            DataShift::Populator.new().prepare_and_assign(attach_to_record_field, record, @attachment)
           else
             # assume its not a has_many and try basic send
             record.send("#{attach_to_record_field}=", @attachment)

data/lib/loaders/paperclip/image_loading.rb

@@ -11,17 +11,17 @@ require 'attachment_loader'
 module DataShift
 
   module ImageLoading
- 
+
+    include DataShift::Logging
     include DataShift::Paperclip 
 
-    
     # Note the paperclip attachment model defines the storage path via something like :
     # 
     # => :path => ":rails_root/public/blah/blahs/:id/:style/:basename.:extension"
     # 
     # Options 
     # 
-    #   See also DataShift::paperclip create_attachment for more options
+    #   See also DataShift::paperclip create_paperclip_attachment for more options
     #   
     #   Example:  Image is a model class with an attachment.
     #             Image table contains a viewable field which can contain other  models,
@@ -30,7 +30,9 @@ module DataShift
     #   :viewable_record 
     #   
     def create_attachment(klass, attachment_path, record = nil, attach_to_record_field = nil, options = {})
-        
+
+      logger.debug("ImageLoading::create_attachment on Class #{klass}")
+
       image_attributes = { :attributes => 
                             { :alt => (options[:alt] || ""),
                               :position => (!options[:position] && record and record.respond_to?(:images)) ? record.images.length : 0
@@ -39,9 +41,9 @@ module DataShift
 
       attachment_options = options.dup.merge(image_attributes)
       
-      #puts "DEBUG : create_attachment options : #{attachment_options.inspect}"
-       
-      super(klass, attachment_path, record, attach_to_record_field, attachment_options)
+      logger.debug("Adding Attachment for #{klass.inspect}")
+
+      create_paperclip_attachment(klass, attachment_path, record, attach_to_record_field, attachment_options)
     end
     
     # Set of file extensions ImageMagik can process so default glob

data/lib/loaders/reporter.rb

@@ -11,10 +11,17 @@ module DataShift
   class Reporter
 
     include DataShift::Logging
-      
-    attr_accessor :processed_object_count, :loaded_objects, :failed_objects
 
-   
+    # actual data rows/objects inbound
+    attr_accessor :processed_object_count
+    alias_method :processed_inbound_count, :processed_object_count
+
+    # DB objects created, updated etc
+    attr_accessor :loaded_objects, :failed_objects
+
+    # actual data rows/objects inbound
+    attr_accessor :success_inbound_count, :failed_inbound_count
+
     def initialize()
       reset
     end
@@ -22,10 +29,11 @@ module DataShift
     def reset()
       @processed_object_count = 0
       @loaded_objects, @failed_objects = [], []
+      @success_inbound_count, @failed_inbound_count = 0,0
     end
 
     def add_loaded_object(object)
-      @loaded_objects << object unless(object.nil? || @loaded_objects.include?(object))
+      @loaded_objects << object.id unless(object.nil? || @loaded_objects.include?(object))
     end
     
     def add_failed_object(object)
@@ -37,12 +45,13 @@ module DataShift
       
       puts "\nProcessing Summary Report"
       puts ">>>>>>>>>>>>>>>>>>>>>>>>>\n"
-      puts "Processed total of #{processed_object_count} #{processed_object_count > 1 ? 'entries' : 'entry'}"
-      puts "#{loaded_objects.size} objects were succesfully processed."
-       
+      puts "Processed total of #{processed_object_count} inbound #{processed_object_count > 1 ? 'entries' : 'entry'}"
+      puts "#{loaded_objects.size}\tdatabase objects were successfully processed."
+      puts "#{success_inbound_count}\tinbound rows were successfully processed."
+
       puts "There were NO failures." if failed_objects.empty?
         
-      puts "WARNING : Check logs : #{failed_objects.size} rows contained errors and #{failed_objects.size} records NOT created." unless failed_objects.empty?
+      puts "WARNING : Check logs : #{failed_objects.size} rows contained errors" unless failed_objects.empty?
     end
     
   end

data/lib/thor/export.thor

@@ -15,16 +15,17 @@
 #
 # => bundle exec thor datashift:export:excel -m <active record class> -r <output_template.xls> -a
 #
+require 'thor_base'
+
+
 require 'datashift'
-  
-# Note, not DataShift, case sensitive, create namespace for command line : datashift
+
+# Note, for thor not DataShift, case sensitive, want namespace for cmd line to be : datashift
 module Datashift
 
           
-  class Export < Thor     
-  
-    include DataShift::Logging
-      
+  class Export < DataShift::DSThorBase
+
     desc "excel", "export any active record model (with optional associations)" 
     method_option :model, :aliases => '-m', :required => true, :desc => "The active record model to export"
     method_option :result, :aliases => '-r', :required => true, :desc => "Create template of model in supplied file"
@@ -32,11 +33,9 @@ module Datashift
     method_option :exclude, :aliases => '-e',  :type => :array, :desc => "Use with -a : Exclude association types. Any from #{DataShift::MethodDetail::supported_types_enum.to_a.inspect}"
     
     def excel()
-     
-      # TODO - We're assuming run from a rails app/top level dir...
-      # ...can we make this more robust ? e.g what about when using active record but not in Rails app, 
-      require File.expand_path('config/environment.rb')
-   
+
+      start_connections
+
       require 'excel_exporter'
 
       model = options[:model]
@@ -44,7 +43,7 @@ module Datashift
      
       logger.info "Datashift: Start Excel export to #{result}"
             
-      klass = ModelMapper::class_from_string(model)  #Kernel.const_get(model)
+      klass = DataShift::ModelMapper::class_from_string(model)  #Kernel.const_get(model)
     
       raise "ERROR: No such Model [#{model}] found - check valid model supplied via -model <Class>" if(klass.nil?)
 
@@ -86,7 +85,7 @@ module Datashift
      
       logger.info "Datashift: Start CSV export to #{result}"
             
-      klass = ModelMapper::class_from_string(model)  #Kernel.const_get(model)
+      klass = DataShift::ModelMapper::class_from_string(model)  #Kernel.const_get(model)
     
       raise "ERROR: No such Model [#{model}] found - check valid model supplied via -model <Class>" if(klass.nil?)
 

data/lib/thor/generate.thor

@@ -46,16 +46,8 @@ module Datashift
       result = options[:result]
      
       logger.info "Datashift: Start Excel template generation in #{result}"
-            
-      begin
-        # support modules e.g "Spree::Property") 
-        klass = ModelMapper::class_from_string(model)  #Kernel.const_get(model)
-      rescue NameError => e
-        puts e
-        raise Thor::Error.new("ERROR: No such Model [#{model}] found - check valid model supplied")
-      end
 
-      raise Thor::Error.new("ERROR: No such Model [#{model}] found - check valid model supplied") unless(klass)
+      klass = DataShift::ModelMapper.class_from_string_or_raise( model )
       
       begin
         gen = DataShift::ExcelGenerator.new(result)

data/lib/thor/import.thor

@@ -44,38 +44,33 @@ module Datashift
       require 'excel_loader'
 
       model = options[:model]
-      begin
-        # support modules e.g "Spree::Property") 
-        klass = ModelMapper::class_from_string(model) 
-      rescue NameError
-        raise "ERROR: No such AR Model found - check valid model supplied with -m <Class>"
-      end
 
-      raise "ERROR: No such AR Model found - check valid model supplied with -m <Class>" if(klass.nil?) 
-      
+      klass = DataShift::ModelMapper.class_from_string_or_raise( model )
+
+      loader_options = { :instance_methods => true }
+
       if(options[:loader])
         begin
      
-          loader_klass = ModelMapper::class_from_string(options[:loader])  
+          loader_klass = DataShift::ModelMapper::class_from_string(options[:loader])
 
           loader = loader_klass.new(klass)
 
           logger.info("INFO: Using loader : #{loader.class}")
         rescue
           logger.error("INFO: No specific #{model}Loader found  - using generic ExcelLoader")
-          loader = DataShift::ExcelLoader.new(klass, true)
+          loader = DataShift::ExcelLoader.new(klass, nil, loader_options)
         end
       else
         logger.info("No Loader specified - using generic ExcelLoader")
-        loader = DataShift::ExcelLoader.new(klass, true)
+        loader = DataShift::ExcelLoader.new(klass, nil, loader_options)
       end
 
-      logger.info("ARGS #{options.inspect}")
-      loader.logger.verbose if(options['verbose'])
+      #TOFIX - multi loggers to file + STDOUT
+      # loader.logger.verbose if(options['verbose'])
       
       loader.configure_from( options[:config] ) if(options[:config])
 
-
       loader.perform_load(options[:input])
     end
     
@@ -96,19 +91,13 @@ module Datashift
       require 'csv_loader'
 
       model = options[:model]
-      begin
-        # support modules e.g "Spree::Property") 
-        klass = ModelMapper::class_from_string(model) 
-      rescue NameError
-        raise "ERROR: No such AR Model found - check valid model supplied with -m <Class>"
-      end
 
-      raise "ERROR: No such AR Model found - check valid model supplied with -m <Class>" if(klass.nil?) 
-      
+      klass = DataShift::ModelMapper.class_from_string_or_raise( model )
 
       loader = DataShift::CsvLoader.new(klass)
-      
-      loader.logger.verbose if(options['verbose'])
+
+      #TOFIX - multi loggers to file + STDOUT
+      # loader.logger.verbose if(options['verbose'])
       
       loader.configure_from( options[:config] ) if(options[:config])
 

data/lib/thor/mapping.thor

@@ -0,0 +1,65 @@
+# Copyright:: (c) Autotelik Media Ltd 2015
+# Author ::   Tom Statter
+# Date ::     Mar 2015
+# License::   MIT.
+#
+# Usage::
+#
+#  To pull Datashift commands into your main application :
+#
+#     require 'datashift'
+#
+#     DataShift::load_commands
+#
+require 'datashift'
+
+# Note, not DataShift, case sensitive, create namespace for command line : datashift
+module Datashift
+
+
+  class Mapping < Thor
+
+    include DataShift::Logging
+
+    desc "template", "Generate a simple mappings template\nInput is treated as the *source* unless otherwise directed"
+
+    method_option :model, :aliases => '-m', :desc => "The active record model to use for mappings"
+    method_option :model_as_dest, :aliases => '-d', :type=> :boolean, :desc => "Set model attributes as destination"
+
+    method_option :excel, :aliases => '-e', :desc => "The excel spreadsheet to use for mappings"
+    method_option :excel_as_dest, :aliases => '-a', :type=> :boolean, :desc => "Set excel headers as destination"
+
+    method_option :result, :aliases => '-r', :required => true, :desc => "Create template of model in supplied file"
+
+    def template()
+
+      # TODO - We're assuming run from a rails app/top level dir...
+      # ...can we make this more robust ? e.g what about when using active record but not in Rails app, 
+      require File.expand_path('config/environment.rb')
+
+      result = options[:result]
+
+      if(File.directory?(result))
+        result = File.join(result, "mapping_template.yaml")
+        puts "Output generated in #{result}"
+      end
+
+      logger.info "Datashift: Starting mapping template generation in #{result}"
+
+      mapper = DataShift::MappingGenerator.new(result)
+
+      model = options[:model]
+
+      mappings = String.new
+
+      mappings += mapper.generate(model, options) unless(model.nil? && options[:excel])
+
+      mappings += mapper.generate_from_excel(options[:excel], options) if(options[:excel])
+
+      File.open(result, 'w') { |f| f << mappings }
+
+    end
+
+  end
+
+end

data/spec/Gemfile

@@ -1,21 +1,23 @@
 source 'https://rubygems.org'
 
-       
+
 # DEFINE WHICH VERSIONS ON VITAL GEMS WE WANT TO TEST WITH
 
-gem 'rails', '4.0.4'
+gem 'rails', '4.1.8'
 
-gem 'spreadsheet'
+gem 'spreadsheet', '1.0.0'
 gem 'paperclip'
+gem 'activesupport'
 
+gem 'database_cleaner'
 
-gem 'rspec'  # Behavior Driven Development (BDD) for Ruby
-gem 'rspec-core'  # RSpec runner and example groups.
-gem 'rspec-expectations'  # RSpec matchers for should and should_not.
-gem 'rspec-mocks'  # RSpec test double framework with stubbing and mocking.
-gem 'rspec-rails'  # RSpec version 2.x for Rails version 3.x.
+gem 'factory_girl_rails'
 
-gem 'sqlite3'
+gem 'rspec'
+gem 'rspec-core'
+gem 'rspec-expectations' # RSpec matchers for should and should_not.
+gem 'rspec-mocks'        # RSpec test double framework with stubbing and mocking.
+gem 'rspec-rails'       # RSpec version 2.x for Rails version 3.x.
 
 # we need both, for JRuby testing of Excel and non JRuby csv
 platform :jruby do
@@ -23,7 +25,7 @@ platform :jruby do
   gem 'activerecord-jdbcsqlite3-adapter'
 end
 
-platform :ruby do 
-  #gem 'activerecord-sqlite3-adapter'
+platform :ruby do
+  gem 'sqlite3'
 end
 

data/spec/Gemfile.lock

@@ -1,129 +1,134 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    actionmailer (4.0.4)
-      actionpack (= 4.0.4)
-      mail (~> 2.5.4)
-    actionpack (4.0.4)
-      activesupport (= 4.0.4)
-      builder (~> 3.1.0)
-      erubis (~> 2.7.0)
+    actionmailer (4.1.8)
+      actionpack (= 4.1.8)
+      actionview (= 4.1.8)
+      mail (~> 2.5, >= 2.5.4)
+    actionpack (4.1.8)
+      actionview (= 4.1.8)
+      activesupport (= 4.1.8)
       rack (~> 1.5.2)
       rack-test (~> 0.6.2)
-    activemodel (4.0.4)
-      activesupport (= 4.0.4)
-      builder (~> 3.1.0)
-    activerecord (4.0.4)
-      activemodel (= 4.0.4)
-      activerecord-deprecated_finders (~> 1.0.2)
-      activesupport (= 4.0.4)
-      arel (~> 4.0.0)
-    activerecord-deprecated_finders (1.0.3)
-    activerecord-jdbc-adapter (1.3.7)
-      activerecord (>= 2.2)
-    activerecord-jdbcsqlite3-adapter (1.3.7)
-      activerecord-jdbc-adapter (~> 1.3.7)
-      jdbc-sqlite3 (~> 3.7.2)
-    activesupport (4.0.4)
+    actionview (4.1.8)
+      activesupport (= 4.1.8)
+      builder (~> 3.1)
+      erubis (~> 2.7.0)
+    activemodel (4.1.8)
+      activesupport (= 4.1.8)
+      builder (~> 3.1)
+    activerecord (4.1.8)
+      activemodel (= 4.1.8)
+      activesupport (= 4.1.8)
+      arel (~> 5.0.0)
+    activesupport (4.1.8)
       i18n (~> 0.6, >= 0.6.9)
-      minitest (~> 4.2)
-      multi_json (~> 1.3)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
       thread_safe (~> 0.1)
-      tzinfo (~> 0.3.37)
-    arel (4.0.2)
-    bouncy-castle-java (1.5.0147)
-    builder (3.1.4)
+      tzinfo (~> 1.1)
+    arel (5.0.1.20140414130214)
+    builder (3.2.2)
     climate_control (0.0.3)
       activesupport (>= 3.0)
-    cocaine (0.5.4)
+    cocaine (0.5.8)
       climate_control (>= 0.0.3, < 1.0)
+    concurrent-ruby (1.0.1)
+    database_cleaner (1.5.1)
     diff-lcs (1.2.5)
     erubis (2.7.0)
-    hike (1.2.3)
-    i18n (0.6.9)
-    jdbc-sqlite3 (3.7.2.1)
-    jruby-openssl (0.9.4)
-      bouncy-castle-java (>= 1.5.0147)
-    mail (2.5.4)
-      mime-types (~> 1.16)
-      treetop (~> 1.4.8)
-    mime-types (1.25.1)
-    minitest (4.7.5)
-    multi_json (1.10.0)
-    paperclip (4.1.1)
-      activemodel (>= 3.0.0)
+    factory_girl (4.5.0)
       activesupport (>= 3.0.0)
-      cocaine (~> 0.5.3)
+    factory_girl_rails (4.6.0)
+      factory_girl (~> 4.5.0)
+      railties (>= 3.0.0)
+    i18n (0.7.0)
+    json (1.8.3)
+    mail (2.6.3)
+      mime-types (>= 1.16, < 3)
+    mime-types (2.99.1)
+    mimemagic (0.3.0)
+    minitest (5.8.4)
+    paperclip (4.3.5)
+      activemodel (>= 3.2.0)
+      activesupport (>= 3.2.0)
+      cocaine (~> 0.5.5)
       mime-types
-    polyglot (0.3.4)
-    rack (1.5.2)
-    rack-test (0.6.2)
+      mimemagic (= 0.3.0)
+    rack (1.5.5)
+    rack-test (0.6.3)
       rack (>= 1.0)
-    rails (4.0.4)
-      actionmailer (= 4.0.4)
-      actionpack (= 4.0.4)
-      activerecord (= 4.0.4)
-      activesupport (= 4.0.4)
+    rails (4.1.8)
+      actionmailer (= 4.1.8)
+      actionpack (= 4.1.8)
+      actionview (= 4.1.8)
+      activemodel (= 4.1.8)
+      activerecord (= 4.1.8)
+      activesupport (= 4.1.8)
       bundler (>= 1.3.0, < 2.0)
-      railties (= 4.0.4)
-      sprockets-rails (~> 2.0.0)
-    railties (4.0.4)
-      actionpack (= 4.0.4)
-      activesupport (= 4.0.4)
+      railties (= 4.1.8)
+      sprockets-rails (~> 2.0)
+    railties (4.1.8)
+      actionpack (= 4.1.8)
+      activesupport (= 4.1.8)
       rake (>= 0.8.7)
       thor (>= 0.18.1, < 2.0)
-    rake (10.3.2)
-    rspec (2.14.1)
-      rspec-core (~> 2.14.0)
-      rspec-expectations (~> 2.14.0)
-      rspec-mocks (~> 2.14.0)
-    rspec-core (2.14.8)
-    rspec-expectations (2.14.5)
-      diff-lcs (>= 1.1.3, < 2.0)
-    rspec-mocks (2.14.6)
-    rspec-rails (2.14.2)
-      actionpack (>= 3.0)
-      activemodel (>= 3.0)
-      activesupport (>= 3.0)
-      railties (>= 3.0)
-      rspec-core (~> 2.14.0)
-      rspec-expectations (~> 2.14.0)
-      rspec-mocks (~> 2.14.0)
-    ruby-ole (1.2.11.7)
-    spreadsheet (0.9.7)
+    rake (11.0.1)
+    rspec (3.4.0)
+      rspec-core (~> 3.4.0)
+      rspec-expectations (~> 3.4.0)
+      rspec-mocks (~> 3.4.0)
+    rspec-core (3.4.4)
+      rspec-support (~> 3.4.0)
+    rspec-expectations (3.4.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.4.0)
+    rspec-mocks (3.4.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.4.0)
+    rspec-rails (3.4.2)
+      actionpack (>= 3.0, < 4.3)
+      activesupport (>= 3.0, < 4.3)
+      railties (>= 3.0, < 4.3)
+      rspec-core (~> 3.4.0)
+      rspec-expectations (~> 3.4.0)
+      rspec-mocks (~> 3.4.0)
+      rspec-support (~> 3.4.0)
+    rspec-support (3.4.1)
+    ruby-ole (1.2.12)
+    spreadsheet (1.0.0)
       ruby-ole (>= 1.0)
-    sprockets (2.12.1)
-      hike (~> 1.2)
-      multi_json (~> 1.0)
-      rack (~> 1.0)
-      tilt (~> 1.1, != 1.3.0)
-    sprockets-rails (2.0.1)
+    sprockets (3.5.2)
+      concurrent-ruby (~> 1.0)
+      rack (> 1, < 3)
+    sprockets-rails (2.3.3)
       actionpack (>= 3.0)
       activesupport (>= 3.0)
-      sprockets (~> 2.8)
-    sqlite3 (1.3.9)
+      sprockets (>= 2.8, < 4.0)
+    sqlite3 (1.3.11)
     thor (0.19.1)
-    thread_safe (0.3.3)
-    thread_safe (0.3.3-java)
-    tilt (1.4.1)
-    treetop (1.4.15)
-      polyglot
-      polyglot (>= 0.3.1)
-    tzinfo (0.3.39)
+    thread_safe (0.3.5)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
 
 PLATFORMS
-  java
   ruby
 
 DEPENDENCIES
   activerecord-jdbcsqlite3-adapter
+  activesupport
+  database_cleaner
+  factory_girl_rails
   jruby-openssl
   paperclip
-  rails (= 4.0.4)
+  rails (= 4.1.8)
   rspec
   rspec-core
   rspec-expectations
   rspec-mocks
   rspec-rails
-  spreadsheet
+  spreadsheet (= 1.0.0)
   sqlite3
+
+BUNDLED WITH
+   1.11.2