datashift 0.40.3 → 0.40.4

data/lib/tasks/generate.thor

@@ -25,32 +25,37 @@ module Datashift
     include DataShift::ThorBehavior
 
     class_option :associations, aliases: '-a',
-                 type: :boolean,
-                 desc: 'Include associations. Can be further refined by :with & :exclude'
+                                type: :boolean,
+                                desc: 'Include associations. Can be further refined by :with & :exclude'
 
     class_option :expand_associations, type: :boolean,
-                 desc: 'Expand association data to multiple columns i.e 1 column per attribute'
+                                       desc: 'Expand association data to multiple columns i.e 1 column per attribute'
 
+    desc 'excel', 'generate a template from an active record model (with optional associations)'
 
-    desc "excel", "generate a template from an active record model (with optional associations)"
+    method_option :model, aliases: '-m', required: true, desc: 'The active record model to export'
+    method_option :result, aliases: '-r', required: true, desc: 'Create template of model in supplied file'
 
-    method_option :model, :aliases => '-m', :required => true, :desc => "The active record model to export"
-    method_option :result, :aliases => '-r', :required => true, :desc => "Create template of model in supplied file"
+    method_option :include_rails, desc: 'Include Rails auto generated columns like :id, created_at, updated_at'
+    method_option :force, aliases: '-f', type: :array, desc: "Inform datashift of columns that are still call-able even though they're non model methods"
 
-    def excel()
+    def excel
       start_connections
 
-      generate( DataShift::ExcelGenerator.new)
+      export_options = { remove_rails: !options[:include_rails] }
+
+      DataShift::Exporters::Configuration.from_hash(export_options)
+
+      generate(DataShift::ExcelGenerator.new)
 
       puts "Datashift: Excel Template COMPLETED to #{options[:result]}"
     end
 
+    desc 'csv', 'generate a template from an active record model (with optional associations)'
+    method_option :model, aliases: '-m', required: true, desc: 'The active record model to export'
+    method_option :result, aliases: '-r', required: true, desc: 'Create template of model in supplied file'
 
-    desc "csv", "generate a template from an active record model (with optional associations)"
-    method_option :model, :aliases => '-m', :required => true, :desc => "The active record model to export"
-    method_option :result, :aliases => '-r', :required => true, :desc => "Create template of model in supplied file"
-
-    def csv()
+    def csv
       start_connections
 
       generate( DataShift::CsvGenerator.new)
@@ -58,27 +63,27 @@ module Datashift
       puts "Datashift: CSV Template COMPLETED to #{options[:result]}"
     end
 
-    desc "db", "Generate a template for every Active Record model"
+    desc 'db', 'Generate a template for every Active Record model'
 
-    method_option :path, :aliases => '-p', :required => true, desc: "Path in which to create export files"
-    method_option :csv, :aliases => '-c', desc: "Generate CSV template instead - Excel is default."
+    method_option :path, aliases: '-p', required: true, desc: 'Path in which to create export files'
+    method_option :csv, aliases: '-c', desc: 'Generate CSV template instead - Excel is default.'
 
-    method_option :prefix_map, :aliases => '-x', type: :hash, :default => {},
-                  desc: "For namespaced tables/models specify the table prefix to module map e.g spree_:Spree"
+    method_option :prefix_map, aliases: '-x', type: :hash, default: {},
+                               desc: 'For namespaced tables/models specify the table prefix to module map e.g spree_:Spree'
 
-    method_option :modules, :aliases => '-m', type: :array, :default => [],
-                  desc: "List of Modules to search for namespaced models"
+    method_option :modules, aliases: '-m', type: :array, default: [],
+                            desc: 'List of Modules to search for namespaced models'
 
-    def db()
+    def db
 
       start_connections
 
       unless File.directory?(options[:path])
         puts "WARNING : No such PATH found #{options[:path]} - trying mkdir"
-        FileUtils::mkdir_p(options[:path])
+        FileUtils.mkdir_p(options[:path])
       end
 
-      generator = options[:csv] ?  DataShift::CsvGenerator.new :  DataShift::ExcelGenerator.new
+      generator = options[:csv] ? DataShift::CsvGenerator.new : DataShift::ExcelGenerator.new
 
       DataShift::Exporters::Configuration.from_hash(options)
 
@@ -93,13 +98,15 @@ module Datashift
           break if(@klass)
         end
 
-        options[:prefix_map].each do |p, m|
-          @klass = DataShift::MapperUtils.table_to_arclass(table.gsub(p, ''), m)
-          break if(@klass)
-        end unless(@klass)
+        unless @klass
+          options[:prefix_map].each do |p, m|
+            @klass = DataShift::MapperUtils.table_to_arclass(table.gsub(p, ''), m)
+            break if(@klass)
+          end
+        end
 
         if(@klass.nil?)
-          puts  "ERROR: No Model found for Table [#{table}] - perhaps a prefix map required?"
+          puts "ERROR: No Model found for Table [#{table}] - perhaps a prefix map required?"
           next
         end
 
@@ -108,15 +115,15 @@ module Datashift
         puts "Datashift: Start export to #{result} for [#{table}]"
         begin
           if(options[:associations])
-            logger.info("Datashift: Generating with associations")
+            logger.info('Datashift: Generating with associations')
             generator.generate_with_associations(result, @klass)
           else
             generator.generate(result, @klass)
           end
-        rescue => e
+        rescue StandardError => e
           puts e
           puts e.backtrace
-          puts "Warning: Error during export, data may be incomplete"
+          puts 'Warning: Error during export, data may be incomplete'
         end
       end
 
@@ -132,27 +139,26 @@ module Datashift
 
         logger.info "Datashift: Starting template generation for #{generater.class.name} to #{result}"
 
-        klass = DataShift::MapperUtils::class_from_string(model)  #Kernel.const_get(model)
+        klass = DataShift::MapperUtils.class_from_string(model) # Kernel.const_get(model)
 
         raise "ERROR: No such Model [#{model}] found - check valid model supplied via -model <Class>" if(klass.nil?)
 
         begin
-
           if(options[:associations])
-            logger.info("Datashift: Generating template including associations")
+            logger.info('Datashift: Generating template including associations')
             generater.generate_with_associations(result, klass)
           else
             generater.generate(result, klass)
           end
-        rescue => e
+        rescue StandardError => e
           puts e
           puts e.backtrace
-          puts "Warning: Error during export, data may be incomplete"
+          puts 'Warning: Error during export, data may be incomplete'
         end
 
       end
 
-    end   # no_commands
+    end # no_commands
 
   end
 

data/lib/tasks/import.thor

@@ -21,7 +21,6 @@ module Datashift
   when we try
       Datashift::Import.new
 
-
     class_option :model, aliases: '-m', required: true, desc: 'The related active record model'
 
     class_option :input, aliases: '-i', required: true, desc: 'The input file'
@@ -33,7 +32,7 @@ module Datashift
     class_option :config, aliases: '-c', desc: 'YAML config file with defaults, over-rides etc'
 =end
 
-    desc "load", "Import data from file for specific active record model"
+    desc 'load', 'Import data from file for specific active record model'
 
     method_option :model, aliases: '-m', required: true, desc: 'The related active record model'
 
@@ -45,22 +44,23 @@ module Datashift
 
     method_option :config, aliases: '-c', desc: 'YAML config file with defaults, over-rides etc'
 
-    def load()
+    method_option :force, aliases: '-f', type: :array, desc: "Call-able columns even though datashift does not recognise them as model methods. Space separated e.g : -f sku price"
+
+    def load
       start_connections
 
       importer = if(options[:loader])
                    logger.info("Attempting to use supplied Loader : #{options[:loader]}")
-                   DataShift::MapperUtils::class_from_string(options[:loader]).new
+                   DataShift::MapperUtils.class_from_string(options[:loader]).new
                  else
-                   logger.info("No Loader specified - finding appropriate Loader for file type")
+                   logger.info('No Loader specified - finding appropriate Loader for file type')
                    DataShift::Loader::Factory.get_loader(options[:input])
                  end
 
       import(importer)
     end
 
-
-    desc "excel", "Import .xls file for specifiec active record model"
+    desc 'excel', 'Import .xls file for specifiec active record model'
 
     method_option :model, aliases: '-m', required: true, desc: 'The related active record model'
 
@@ -72,22 +72,24 @@ module Datashift
 
     method_option :config, aliases: '-c', desc: 'YAML config file with defaults, over-rides etc'
 
-    def excel()
+    method_option :force, aliases: '-f', type: :array, desc: "Call-able columns even though datashift does not recognise them as model methods. Space separated e.g : -f sku price"
+
+    def excel
 
       start_connections
 
       importer = if(options[:loader])
                    logger.info("Attempting to use supplied Loader : #{options[:loader]}")
-                   DataShift::MapperUtils::class_from_string(options[:loader]).new
+                   DataShift::MapperUtils.class_from_string(options[:loader]).new
                  else
-                   logger.info("No Loader specified - using standard Excel Loader")
+                   logger.info('No Loader specified - using standard Excel Loader')
                    DataShift::ExcelLoader.new
                  end
 
       import(importer)
     end
 
-    desc "csv", "Import CSV file for specified active record model"
+    desc 'csv', 'Import CSV file for specified active record model'
 
     method_option :model, aliases: '-m', required: true, desc: 'The related active record model'
 
@@ -99,15 +101,17 @@ module Datashift
 
     method_option :config, aliases: '-c', desc: 'YAML config file with defaults, over-rides etc'
 
-    def csv()
+    method_option :force, aliases: '-f', type: :array, desc: "Call-able columns even though datashift does not recognise them as model methods. Space separated e.g : -f sku price"
+
+    def csv
 
       start_connections
 
       importer = if(options[:loader])
                    logger.info("Attempting to use supplied Loader : #{options[:loader]}")
-                   DataShift::MapperUtils::class_from_string(options[:loader]).new
+                   DataShift::MapperUtils.class_from_string(options[:loader]).new
                  else
-                   logger.info("No Loader specified - using standard Csv Loader")
+                   logger.info('No Loader specified - using standard Csv Loader')
                    DataShift::CsvLoader.new
                  end
 
@@ -121,6 +125,8 @@ module Datashift
 
         importer.configure_from( options[:config] ) if(options[:config])
 
+        DataShift::Configuration.call.force_inclusion_of_columns = options[:force]
+
         importer.run(options[:input], options[:model])
       end
 

data/lib/tasks/paperclip.thor

@@ -12,23 +12,23 @@ require_relative 'thor_behaviour'
 
 # Note, not DataShift, case sensitive, create namespace for command line : datashift
 module Datashift
-         
+
   class Paperclip < Thor
 
     include DataShift::ThorBehavior
 
-    desc "attach", "Create paperclip attachments and attach to a Model from files in a directory.
+    desc 'attach', "Create paperclip attachments and attach to a Model from files in a directory.
     This is specifically for the use case where the paperclip attachments are stored in a class, such as Image, Icon, Asset,
     and this class has a relationship, such as belongs_to, with another class, such as Product, User, Document.
- 
+
     Each matching file is used to create an instance of the paperclip attachment, given by :attachment_klass.
-    
+
     The class with the relationship, can be specified via :attach_to_klass
 
-      Examples     
+      Examples
         Owner has_many pdfs and mp3 files as Digitals .... :attachment_klass = Digital and :attach_to_klass = Owner
         User has a single Image used as an avatar ... attachment_klass = Image and :attach_to_klass = User
-    
+
     The file name is used to lookup the instance of :attach_to_klass to assign the new attachment to, via :attach_to_find_by_field
 
     So say we have a file called smithj_avatar.gif, and we want to lookup Users by login
@@ -37,60 +37,58 @@ module Datashift
 
     Once instance of :attach_to_klass found, the new attachment is assigned.
 
-     The attribute to assign new attachment to is gtiven by :attach_to_field    
-  
+     The attribute to assign new attachment to is gtiven by :attach_to_field
+
      Examples
         :attach_to_field => digitals  : Owner.digitals = attachment(Digital)
         :attach_to_field => avatar    : User.avatar = attachment(Image)"
-    
 
     # :dummy => dummy run without actual saving to DB
-    method_option :input, :aliases => '-i', :required => true, :desc => "The input path containing images "
-    
-    method_option :glob, :aliases => '-g',  :desc => 'The glob to use to find files e.g. \'{*.jpg,*.gif,*.png}\' '
-    method_option :recursive, :aliases => '-r', :type => :boolean, :desc => "Scan sub directories of input for images"
-     
-    method_option :attachment_klass, :required => true, :aliases => '-a', :desc => "Ruby Class name of the Attachment e.g Image, Icon"
-    
-    method_option :attach_to_klass, :required => true, :aliases => '-k', :desc => "A class that has a relationship with the attachment (has_many, has_one, belongs_to)"     
-    method_option :attach_to_find_by_field, :required => true, :aliases => '-l', :desc => "The field to use to find the :attach_to_klass record"
-    method_option :attach_to_field, :required => true, :aliases => '-f', :desc => "Attachment belongs to field e.g Product.image, Blog.digital"
-  
-    
-      # => :attach_to_find_by_field    
-      #       For the :attach_to_klass, this is the field used to search for the parent
-      #       object to assign the new attachment to.
-      #     Examples     
-      #       Owner has a unique 'name' field ... :attach_to_find_by_field = :name
-      #       User has a unique  'login' field  ... :attach_to_klass = :login
-      #
-      # => :attach_to_field    
-      #       Attribute/association to assign attachment to on :attach_to_klass.
-      #      Examples
-      #         :attach_to_field => digitals  : Owner.digitals = attachment
-      #         :attach_to_field => avatar    : User.avatar = attachment
-      
-    method_option :split_file_name_on,  :type => :string,
-                  :desc => "delimiter to progressivley split file_name for lookup", :default => ' '
-
-    method_option :case_sensitive, :type => :boolean, :desc => "Use case sensitive where clause to find :attach_to_klass"
-    method_option :use_like, :type => :boolean, :desc => "Use :lookup_field LIKE 'string%' instead of :lookup_field = 'string' in where clauses to find :attach_to_klass"
-
-    method_option :skip_when_assoc, :aliases => '-x', :type => :boolean, :desc => "Do not process if :attach_to_klass already has an attachment"
-    
-    method_option :verbose, :aliases => '-v', :type => :boolean, :desc => "Verbose logging"
-
-    def attach()
+    method_option :input, aliases: '-i', required: true, desc: 'The input path containing images '
+
+    method_option :glob, aliases: '-g',  desc: 'The glob to use to find files e.g. \'{*.jpg,*.gif,*.png}\' '
+    method_option :recursive, aliases: '-r', type: :boolean, desc: 'Scan sub directories of input for images'
+
+    method_option :attachment_klass, required: true, aliases: '-a', desc: 'Ruby Class name of the Attachment e.g Image, Icon'
+
+    method_option :attach_to_klass, required: true, aliases: '-k', desc: 'A class that has a relationship with the attachment (has_many, has_one, belongs_to)'
+    method_option :attach_to_find_by_field, required: true, aliases: '-l', desc: 'The field to use to find the :attach_to_klass record'
+    method_option :attach_to_field, required: true, aliases: '-f', desc: 'Attachment belongs to field e.g Product.image, Blog.digital'
+
+    # => :attach_to_find_by_field
+    #       For the :attach_to_klass, this is the field used to search for the parent
+    #       object to assign the new attachment to.
+    #     Examples
+    #       Owner has a unique 'name' field ... :attach_to_find_by_field = :name
+    #       User has a unique  'login' field  ... :attach_to_klass = :login
+    #
+    # => :attach_to_field
+    #       Attribute/association to assign attachment to on :attach_to_klass.
+    #      Examples
+    #         :attach_to_field => digitals  : Owner.digitals = attachment
+    #         :attach_to_field => avatar    : User.avatar = attachment
+
+    method_option :split_file_name_on, type: :string,
+                                       desc: 'delimiter to progressivley split file_name for lookup', default: ' '
+
+    method_option :case_sensitive, type: :boolean, desc: 'Use case sensitive where clause to find :attach_to_klass'
+    method_option :use_like, type: :boolean, desc: "Use :lookup_field LIKE 'string%' instead of :lookup_field = 'string' in where clauses to find :attach_to_klass"
+
+    method_option :skip_when_assoc, aliases: '-x', type: :boolean, desc: 'Do not process if :attach_to_klass already has an attachment'
+
+    method_option :verbose, aliases: '-v', type: :boolean, desc: 'Verbose logging'
+
+    def attach
 
       @attachment_path = options[:input]
-      
+
       unless(File.exist?(@attachment_path))
         puts "ERROR: Supplied Path [#{@attachment_path}] not accesible"
         exit(-1)
       end
 
       start_connections
-      
+
       require 'paperclip/attachment_loader'
 
       puts "Using Field #{options[:attach_to_find_by_field]} to lookup matching [#{options[:attach_to_klass]}]"
@@ -104,5 +102,5 @@ module Datashift
       loader.run(@attachment_path, options[:attachment_klass])
     end
   end
-  
+
 end

data/lib/tasks/thor_behaviour.rb

@@ -13,7 +13,7 @@ module DataShift
       if File.exist?(File.expand_path('config/environment.rb'))
         begin
           require File.expand_path('config/environment.rb')
-        rescue => e
+        rescue StandardError => e
           logger.error("Failed to initialise ActiveRecord : #{e.message}")
           raise ConnectionError, "Failed to initialise ActiveRecord : #{e.message}"
         end

data/lib/tasks/to_convert_to_thor/db_tasks.rake

@@ -8,9 +8,7 @@ namespace :datashift do
     task :purge, [:exclude_system_tables] => [:environment] do |_t, args|
       require 'highline/import'
 
-      if Rails.env.production?
-        agree('WARNING: In Production database, REALLY PURGE ? [y]:')
-      end
+      agree('WARNING: In Production database, REALLY PURGE ? [y]:') if Rails.env.production?
 
       config = ActiveRecord::Base.configurations[Rails.env || 'development']
       case config['adapter']

data/lib/tasks/tools.thor

@@ -5,22 +5,22 @@
 #
 #
 require 'datashift'
-  
+
 # Note, not DataShift, case sensitive, create namespace for command line : datashift
 module Datashift
 
-  class Tools < Thor     
-  
+  class Tools < Thor
+
     include DataShift::Logging
 
     desc 'file_rename', 'Clone a folder of files, consistently renaming each in the process'
 
-    method_option :path, :aliases => '-p', required: true, desc: "The path to the original files"
-    method_option :output, :aliases => '-o', required: true, desc: "The resulting zip file name"
-    method_option :offset,  required: false, type: :numeric, desc: "A numeric offset to add to the file name"
-    method_option :width,  required: false, type: :numeric, desc: "A numeric width tp pad the file name"
-    method_option :prefix,  required: false, desc: "A strign prefix to add to file name"
-    method_option :commit,  required: false, type: :boolean, desc: "Actually perform copy"
+    method_option :path, aliases: '-p', required: true, desc: 'The path to the original files'
+    method_option :output, aliases: '-o', required: true, desc: 'The resulting zip file name'
+    method_option :offset, required: false, type: :numeric, desc: 'A numeric offset to add to the file name'
+    method_option :width, required: false, type: :numeric, desc: 'A numeric width tp pad the file name'
+    method_option :prefix,  required: false, desc: 'A strign prefix to add to file name'
+    method_option :commit,  required: false, type: :boolean, desc: 'Actually perform copy'
 
     def file_rename
 
@@ -33,7 +33,7 @@ module Datashift
           id = base_name.slice!(/\w+/)
 
           id = id.to_i + options[:offset].to_i if options[:offset]
-          id = "%0#{width}d" % id.to_i if options[:width]
+          id = format("%0#{width}d", id.to_i) if options[:width]
           id = options[:prefix] + id.to_s if options[:prefix]
 
           destination = File.join( options[:output], "#{id}#{base_name}")
@@ -44,13 +44,13 @@ module Datashift
       end
     end
 
-    desc "zip", "Create zip of files" 
+    desc 'zip', 'Create zip of files'
+
+    method_option :path, aliases: '-p', required: true, desc: 'The path to the digital files'
+    method_option :output, aliases: '-o', required: true, desc: 'The resulting zip file name'
 
-    method_option :path, :aliases => '-p', required: true, desc: "The path to the digital files"
-    method_option :output, :aliases => '-o', required: true, desc: "The resulting zip file name"
- 
     def zip
-     
+
       require 'zip/zip'
       require 'zip/zipfilesystem'
 
@@ -63,36 +63,36 @@ module Datashift
         end
       end
     end
-    
-    desc "zip_matching", "Create zip of matching digital files e.g zip up pdf, jpg and png versions of a file" 
-
-    method_option :path, :aliases => '-p', required: true, desc: "The path to the digital files"
-    method_option :results, :aliases => '-r', required: true, desc: "The path to store resulting zip files"
- 
-    def zip_matching()
-     
+
+    desc 'zip_matching', 'Create zip of matching digital files e.g zip up pdf, jpg and png versions of a file'
+
+    method_option :path, aliases: '-p', required: true, desc: 'The path to the digital files'
+    method_option :results, aliases: '-r', required: true, desc: 'The path to store resulting zip files'
+
+    def zip_matching
+
       require 'zip/zip'
       require 'zip/zipfilesystem'
 
       ready_to_zip = {}
       Dir[File.join(options[:path], '**', '*.*')].each do |p|
         next if File.directory? p
-        
+
         basename = File.basename(p, '.*')
-        ready_to_zip[basename] ||= []       
-        ready_to_zip[basename] << p  
-      end 
-      
+        ready_to_zip[basename] ||= []
+        ready_to_zip[basename] << p
+      end
+
       output = options[:results]
-      
-      FileUtils::mkdir_p(output) unless File.exist?(output)
-      
+
+      FileUtils.mkdir_p(output) unless File.exist?(output)
+
       puts "Creating #{ready_to_zip.keys.size} new zips"
       ready_to_zip.each do |basename, paths|
-      
-        z= File.join(output, basename + '.zip')
+
+        z = File.join(output, basename + '.zip')
         puts "zipping to #{z}"
-        
+
         Zip::ZipOutputStream.open(z) do |zos|
           paths.each do |file|
             zos.put_next_entry(File.basename(file))
@@ -100,10 +100,9 @@ module Datashift
           end
         end
       end
-      
-    end   
-    
+
+    end
+
   end
 
 end
-