datashift 0.16.0 → 0.40.0

data/lib/loaders/paperclip/image_loading.rb

@@ -3,7 +3,7 @@
 # Date ::     June 2012
 # License::   MIT. Free, Open Source.
 #
-# => Provides facilities for bulk uploading/exporting attachments provided by PaperClip 
+# => Provides facilities for bulk uploading/exporting attachments provided by PaperClip
 # gem
 require 'datashift_paperclip'
 require 'attachment_loader'
@@ -13,62 +13,67 @@ module DataShift
   module ImageLoading
 
     include DataShift::Logging
-    include DataShift::Paperclip 
+    include DataShift::Paperclip
 
     # Note the paperclip attachment model defines the storage path via something like :
-    # 
+    #
     # => :path => ":rails_root/public/blah/blahs/:id/:style/:basename.:extension"
-    # 
-    # Options 
-    # 
+    #
+    # Options
+    #
     #   See also DataShift::paperclip create_paperclip_attachment for more options
-    #   
+    #
     #   Example:  Image is a model class with an attachment.
     #             Image table contains a viewable field which can contain other  models,
     #             such as Product, User etc all of which can have an Image
-    #             
-    #   :viewable_record 
-    #   
+    #
+    #   :viewable_record
+    #
     def create_attachment(klass, attachment_path, record = nil, attach_to_record_field = nil, options = {})
 
       logger.debug("ImageLoading::create_attachment on Class #{klass}")
 
-      image_attributes = { :attributes => 
-                            { :alt => (options[:alt] || ""),
-                              :position => (!options[:position] && record and record.respond_to?(:images)) ? record.images.length : 0
+      image_attributes = { attributes: { alt: (options[:alt] || ''),
+                                         position: (!options[:position] && record && record.respond_to?(:images)) ? record.images.length : 0
                             }
       }
 
       attachment_options = options.dup.merge(image_attributes)
-      
+
       logger.debug("Adding Attachment for #{klass.inspect}")
 
-      create_paperclip_attachment(klass, attachment_path, record, attach_to_record_field, attachment_options)
+      attachment = create_paperclip_attachment(klass, attachment_path, attachment_options)
+
+      if(attachment && attach_to_record_field)
+        populator = DataShift::Populator.new
+        populator.prepare_and_assign(attach_to_record_field, record, attachment)
+      end
+
     end
-    
+
     # Set of file extensions ImageMagik can process so default glob
     # we use to find image files within directories
     def self.image_magik_glob
-      @im_glob ||= %w{*.3FR *.AAI *.AI *.ART *.ARW  *.AVI *.AVS *.BGR *.BGRA
-                  *.BIE *.BMP *.BMP2  *.BMP3  *.BRF *.CAL *.CALS *.CANVAS
-                  *.CIN *.CIP *.CLIP *.CMYK *.CMYKA *.CR2 *.CRW *.CUR *.CUT *.DCM *.DCR *.DCX
-                  *.DDS *.DFONT *.DJVU *.DNG  *.DOT *.DPS *.DPX
-                  *.EMF *.EPDF  *.EPI *.EPS *.EPS2 *.EPS3  *.EPSF *.EPSI
-                  *.EPT *.EPT2 *.EPT3 *.ERF *.EXR *.FITS *.FPX  *.FTS *.G3 *.GIF *.GIF87
-                  *.GRAY *.HALD  *.HDR *.HRZ  *.ICB *.ICO *.ICON *.IPL
-                  *.ISOBRL *.J2C *.JBG  *.JBIG *.JNG *.JP2 *.JPC *.JPEG *.JPG *.JPX  *.K25 *.KDC
-                  *.LABEL *.M2V  *.M4V  *.MAC *.MAP  *.MAT *.MATTE *.MIFF *.MNG  *.MONO
-                  *.MOV *.MP4 *.MPC *.MPEG *.MPG *.MRW *.MSL *.MSVG *.MTV *.MVG *.NEF *.ORF *.OTB *.OTF *.PAL *.PALM
-                  *.PAM *.PBM  *.PCD *.PCDS *.PCL *.PCT *.PCX *.PDB *.PDF *.PDFA  *.PEF
-                  *.PES *.PFA *.PFB *.PFM *.PGM  *.PGX *.PICON *.PICT *.PIX *.PJPEG *.PLASMA
-                  *.PNG *.PNG24  *.PNG32 *.PNG8 *.PNM *.PPM *.PS *.PS2 *.PS3 *.PSB *.PSD *.PTIF *.PWP *.RAF *.RAS *.RGB
-                  *.RGBA *.RGBO *.RLA *.RLE *.SCR *.SCT *.SFW *.SGI *.SR2 *.SRF
-                  *.SUN *.SVG *.SVGZ *.TGA *.TIFF *.TIFF64 *.TILE *.TIM *.TTC *.TTF *.UBRL *.UIL *.UYVY *.VDA  *.VICAR
-                  *.VID *.VIFF  *.VST *.WBMP *.WEBP  *.WMF *.WMV *.WMZ *.WPG  *.X3F
-                  *.XBM *.XC *.XCF *.XPM *.XPS *.XV *.XWD *.YCbCr *.YCbCrA *.YUV
-      }
+      @im_glob ||= %w(*.3FR *.AAI *.AI *.ART *.ARW  *.AVI *.AVS *.BGR *.BGRA
+                      *.BIE *.BMP *.BMP2  *.BMP3  *.BRF *.CAL *.CALS *.CANVAS
+                      *.CIN *.CIP *.CLIP *.CMYK *.CMYKA *.CR2 *.CRW *.CUR *.CUT *.DCM *.DCR *.DCX
+                      *.DDS *.DFONT *.DJVU *.DNG  *.DOT *.DPS *.DPX
+                      *.EMF *.EPDF  *.EPI *.EPS *.EPS2 *.EPS3  *.EPSF *.EPSI
+                      *.EPT *.EPT2 *.EPT3 *.ERF *.EXR *.FITS *.FPX  *.FTS *.G3 *.GIF *.GIF87
+                      *.GRAY *.HALD  *.HDR *.HRZ  *.ICB *.ICO *.ICON *.IPL
+                      *.ISOBRL *.J2C *.JBG  *.JBIG *.JNG *.JP2 *.JPC *.JPEG *.JPG *.JPX  *.K25 *.KDC
+                      *.LABEL *.M2V  *.M4V  *.MAC *.MAP  *.MAT *.MATTE *.MIFF *.MNG  *.MONO
+                      *.MOV *.MP4 *.MPC *.MPEG *.MPG *.MRW *.MSL *.MSVG *.MTV *.MVG *.NEF *.ORF *.OTB *.OTF *.PAL *.PALM
+                      *.PAM *.PBM  *.PCD *.PCDS *.PCL *.PCT *.PCX *.PDB *.PDF *.PDFA  *.PEF
+                      *.PES *.PFA *.PFB *.PFM *.PGM  *.PGX *.PICON *.PICT *.PIX *.PJPEG *.PLASMA
+                      *.PNG *.PNG24  *.PNG32 *.PNG8 *.PNM *.PPM *.PS *.PS2 *.PS3 *.PSB *.PSD *.PTIF *.PWP *.RAF *.RAS *.RGB
+                      *.RGBA *.RGBO *.RLA *.RLE *.SCR *.SCT *.SFW *.SGI *.SR2 *.SRF
+                      *.SUN *.SVG *.SVGZ *.TGA *.TIFF *.TIFF64 *.TILE *.TIM *.TTC *.TTF *.UBRL *.UIL *.UYVY *.VDA  *.VICAR
+                      *.VID *.VIFF  *.VST *.WBMP *.WEBP  *.WMF *.WMV *.WMZ *.WPG  *.X3F
+                      *.XBM *.XC *.XCF *.XPM *.XPS *.XV *.XWD *.YCbCr *.YCbCrA *.YUV
+                   )
       "{#{@im_glob.join(',')}}"
     end
   end
-      
+
 end

data/lib/loaders/reporters/basic_stdout_reporter.rb

@@ -0,0 +1,40 @@
+# Copyright:: (c) Autotelik Media Ltd 2016
+# Author ::   Tom Statter
+# License::   MIT
+#
+# Details::   Very basic report to dump out loading report stats
+#
+module DataShift
+  module Reporters
+    class BasicStdoutReporter < Reporter
+
+      def initialize(progress_monitor)
+        super progress_monitor
+      end
+
+      def report
+        loaded_objects = progress_monitor.loaded_objects
+
+        loaded_objects.compact! if loaded_objects
+
+        inbound_str = progress_monitor.processed_object_count > 1 ? 'entries' : 'entry'
+
+        puts "\nProcessing Summary Report"
+        puts ">>>>>>>>>>>>>>>>>>>>>>>>>\n"
+        puts "Processed total of #{progress_monitor.processed_object_count} inbound #{inbound_str}"
+        puts "#{loaded_objects.size}\tdatabase objects were successfully processed."
+        puts "#{progress_monitor.success_inbound_count}\tinbound rows were successfully processed."
+
+        failed_inbound_count = progress_monitor.failed_inbound_count
+
+        if failed_inbound_count == 0
+          puts 'There were NO failures.'
+        else
+          puts "WARNING : There were Failures - Check logs\n#{failed_inbound_count} rows contained errors"
+          puts "#{progress_monitor.failed_objects.size} objects could not be saved to DB"
+        end
+      end
+
+    end
+  end
+end

data/lib/loaders/reporters/reporter.rb

@@ -0,0 +1,26 @@
+# Copyright:: (c) Autotelik Media Ltd 2016
+# Author ::   Tom Statter
+# License::   MIT
+#
+# Details::   Base class for reporters on loading stats
+#
+module DataShift
+  module Reporters
+    class Reporter
+
+      include DataShift::Logging
+
+      # Holds the actual data r.e data rows/objects inbound
+      attr_accessor :progress_monitor
+
+      def initialize(progress_monitor = DataShift::ProgressMonitor.new)
+        @progress_monitor = progress_monitor
+      end
+
+      # an abstract method
+      def report
+      end
+
+    end
+  end
+end

data/lib/tasks/config.thor

@@ -0,0 +1,65 @@
+# Copyright:: (c) Autotelik Media Ltd 2016
+# Author ::   Tom Statter
+# Date ::     Mar 2016
+# License::   MIT.
+#
+#
+require 'thor'
+
+# Note, not DataShift, case sensitive, create namespace for command line : datashift
+
+module Datashift
+
+  class Config < Thor
+
+    DEFAULT_IMPORT_TEMPLTE ||= "import_mapping_template.yaml".freeze
+
+    include DataShift::Logging
+
+    desc "import", "Generate an Import configuration template (YAML)"
+
+    method_option :model, aliases: '-m', required: true, desc: "The active record model to use for mappings"
+
+    method_option :result, aliases: '-r', required: true,
+                  desc: "Path or file to create resulting YAML config\nIf PATH, filename is [#{DEFAULT_IMPORT_TEMPLTE}]"
+
+    def import()
+
+      start_connections
+
+      result = options[:result]
+
+      if(File.directory?(result))
+        result = File.join(result, DEFAULT_IMPORT_TEMPLTE)
+      end
+
+      logger.info "Datashift: Starting Import mapping template generation to [#{result}]"
+
+      mapper = DataShift::ConfigGenerator.new
+
+      puts "Creating new configuration file : [#{result}]"
+      mapper.write_import(result, options[:model], options)
+
+    end
+
+    no_commands do
+
+      def start_connections
+
+        if File.exist?(File.expand_path('config/environment.rb'))
+          begin
+            require File.expand_path('config/environment.rb')
+          rescue => e
+            logger.error("Failed to initialise ActiveRecord : #{e.message}")
+            raise ConnectionError.new("Failed to initialise ActiveRecord : #{e.message}")
+          end
+
+        else
+          raise PathError.new('No config/environment.rb found - cannot initialise ActiveRecord')
+        end
+      end
+
+    end
+
+  end
+end

data/lib/tasks/export.thor

@@ -0,0 +1,192 @@
+# Copyright:: (c) Autotelik Media Ltd 2016
+# Author ::   Tom Statter
+# Date ::     March 2016
+# License::   MIT.
+#
+# Usage::
+#
+#
+require 'thor'
+
+# Note, for thor not DataShift, case sensitive, want namespace for cmd line to be : datashift
+module Datashift
+
+  class Export < Thor
+
+    include DataShift::Logging
+
+    class_option :associations, aliases: '-a',
+                 type: :boolean,
+                 desc: 'Include associations. Can be further refined by :with & :exclude'
+
+    class_option :expand_associations, type: :boolean,
+                 desc: 'Expand association data to multiple columns i.e 1 column per attribute'
+
+    class_option :methods, type: :array,
+                 desc: 'List of additional methods to call on model, useful for situations like delegated methods'
+
+    class_option :with, type: :array,
+                 desc: "Restrict association types. Choose from #{DataShift::ModelMethod.supported_types_enum.inspect}"
+
+    class_option :exclude, type: :array,
+                 desc: "Exclude association types. Choose from #{DataShift::ModelMethod.supported_types_enum.inspect}"
+
+    class_option :remove,  type: :array,
+                 desc: "Don't include this list of supplied fields"
+
+    class_option :remove_rails, type: :boolean,
+                 desc: "Remove standard Rails cols :  #{DataShift::Configuration.rails_columns.inspect}"
+
+    class_option :json, type: :boolean,
+                 desc: 'Export association data as json rather than hash'
+
+    desc "excel", "export any active record model (with optional associations)"
+
+    method_option :model, :aliases => '-m', :required => true, desc: "The active record model to export"
+    method_option :result, :aliases => '-r', :required => true, desc: "Create template of model in supplied file"
+
+    method_option :sheet_name, :type => :string, desc: "Name to use for Excel worksheet instead of model name"
+
+    def excel()
+      start_connections
+
+      export(DataShift::ExcelExporter.new)
+
+      puts "Datashift: Excel export COMPLETED to #{options[:result]}"
+    end
+
+
+    desc "csv", "export any active record model (with optional associations)"
+
+    method_option :model, :aliases => '-m', :required => true, desc: "The active record model to export"
+    method_option :result, :aliases => '-r', :required => true, desc: "Create template of model in supplied file"
+
+    def csv()
+      start_connections
+
+      export(DataShift::CsvExporter.new)
+
+      puts "Datashift: CSV export COMPLETED to #{options[:result]}"
+    end
+
+
+    desc "db", "Export every Active Record model"
+
+    method_option :model, :aliases => '-m', :required => true, desc: "The active record model to export"
+    method_option :result, :aliases => '-r', :required => true, desc: "Create template of model in supplied file"
+
+    method_option :csv, :aliases => '-c', desc: "Export to CSV instead - Excel is default."
+
+    method_option :prefix_map, :aliases => '-x', type: :hash, :default => {},
+                  desc: "For namespaced tables/models specify the table prefix to module map e.g spree_:Spree"
+
+    method_option :modules, :aliases => '-m', type: :array, :default => [],
+                  desc: "List of Modules to search for namespaced models"
+
+    def db()
+
+      start_connections
+
+      FileUtils::mkdir_p(options[:result]) unless File.directory?(options[:result])
+
+      raise "WARNING : One file per model - results expects a DIRECTORY" unless File.directory?(options[:result])
+
+      exporter = options[:csv] ?  DataShift::CsvExporter.new :  DataShift::ExcelExporter.new
+
+      DataShift::Exporters::Configuration.from_hash(options)
+
+      ext = options[:csv] ? '.csv' : '.xls'
+
+      modules = [nil] + options[:modules]
+
+      ActiveRecord::Base.connection.tables.each do |table|
+
+        logger.info("Starting Export process for Table #{table}")
+
+        modules.each do |m|
+          @klass = DataShift::MapperUtils.table_to_arclass(table, m)
+          break if(@klass)
+        end
+
+        options[:prefix_map].each do |p, m|
+          @klass = DataShift::MapperUtils.table_to_arclass(table.gsub(p, ''), m)
+          break if(@klass)
+        end unless(@klass)
+
+        if(@klass.nil?)
+          puts  "ERROR: No Model found for Table [#{table}] - perhaps check modules/prefixes"
+          next
+        end
+
+        result = File.join(options[:result], "#{table}#{ext}")
+
+        puts "Datashift: Start export to #{result} for [#{table}]"
+
+        logger.info("Starting Export to #{result} for [#{@klass}]")
+
+        begin
+
+          if(options[:associations])
+            logger.info("Datashift: Exporting with associations")
+            exporter.export_with_associations(result, @klass, @klass.all)
+          else
+            exporter.export(result, @klass.all, :sheet_name => @klass.name)
+          end
+        rescue => e
+          puts e
+          puts e.backtrace
+          puts "Warning: Error during export, data may be incomplete"
+        end
+      end
+    end
+
+    no_commands do
+
+      def start_connections
+
+        if File.exist?(File.expand_path('config/environment.rb'))
+          begin
+            require File.expand_path('config/environment.rb')
+          rescue => e
+            logger.error("Failed to initialise ActiveRecord : #{e.message}")
+            raise ConnectionError.new("Failed to initialise ActiveRecord : #{e.message}")
+          end
+
+        else
+          raise PathError.new('No config/environment.rb found - cannot initialise ActiveRecord')
+        end
+      end
+
+      def export(exporter)
+        model = options[:model]
+        result = options[:result]
+
+        DataShift::Configuration.from_hash(options)
+
+        logger.info "Datashift: Starting export with #{exporter.class.name} to #{result}"
+
+        klass = DataShift::MapperUtils::class_from_string(model)  #Kernel.const_get(model)
+
+        raise "ERROR: No such Model [#{model}] found - check valid model supplied via -model <Class>" if(klass.nil?)
+
+        begin
+
+          if(options[:associations])
+            logger.info("Datashift: Exporting with associations")
+            exporter.export_with_associations(result, klass, klass.all)
+          else
+            exporter.export(result, klass.all, options)
+          end
+        rescue => e
+          puts e
+          puts e.backtrace
+          puts "Warning: Error during export, data may be incomplete"
+        end
+
+      end
+
+    end   # no_commands
+
+  end
+
+end

data/lib/tasks/generate.thor

@@ -0,0 +1,190 @@
+# Copyright:: (c) Autotelik Media Ltd 2016
+# Author ::   Tom Statter
+# Date ::     Mar 2016
+# License::   MIT.
+#
+# Usage::
+#
+#  To pull Datashift commands into your main application :
+#
+#     require 'datashift'
+#
+#     DataShift::load_commands
+#
+#
+require 'thor'
+
+require 'csv_generator'
+
+# Note, not DataShift, case sensitive, create namespace for command line : datashift
+
+module Datashift
+
+  class Generate  < Thor
+
+    include DataShift::Logging
+
+    class_option :associations, aliases: '-a',
+                 type: :boolean,
+                 desc: 'Include associations. Can be further refined by :with & :exclude'
+
+    class_option :expand_associations, type: :boolean,
+                 desc: 'Expand association data to multiple columns i.e 1 column per attribute'
+
+    class_option :methods, type: :array,
+                 desc: 'List of additional methods to call on model, useful for situations like delegated methods'
+
+    class_option :with, type: :array,
+                 desc: "Restrict association types. Choose from #{DataShift::ModelMethod.supported_types_enum.inspect}"
+
+    class_option :exclude, type: :array,
+                 desc: "Exclude association types. Choose from #{DataShift::ModelMethod.supported_types_enum.inspect}"
+
+    class_option :remove,  type: :array,
+                 desc: "Don't include this list of supplied fields"
+
+    class_option :remove_rails, type: :boolean,
+                 desc: "Remove standard Rails cols :  #{DataShift::Configuration.rails_columns.inspect}"
+
+    class_option :json, type: :boolean,
+                 desc: 'Export association data as json rather than hash'
+
+    desc "excel", "generate a template from an active record model (with optional associations)"
+
+    method_option :model, :aliases => '-m', :required => true, :desc => "The active record model to export"
+    method_option :result, :aliases => '-r', :required => true, :desc => "Create template of model in supplied file"
+
+    def excel()
+      start_connections
+
+      generate( DataShift::ExcelGenerator.new)
+
+      puts "Datashift: Excel Template COMPLETED to #{options[:result]}"
+    end
+
+
+    desc "csv", "generate a template from an active record model (with optional associations)"
+    method_option :model, :aliases => '-m', :required => true, :desc => "The active record model to export"
+    method_option :result, :aliases => '-r', :required => true, :desc => "Create template of model in supplied file"
+
+    def csv()
+      start_connections
+
+      generate( DataShift::CsvGenerator.new)
+
+      puts "Datashift: CSV Template COMPLETED to #{options[:result]}"
+    end
+
+    desc "db", "Generate a template for every Active Record model"
+
+    method_option :path, :aliases => '-p', :required => true, desc: "Path in which to create export files"
+    method_option :csv, :aliases => '-c', desc: "Generate CSV template instead - Excel is default."
+
+    method_option :prefix_map, :aliases => '-x', type: :hash, :default => {},
+                  desc: "For namespaced tables/models specify the table prefix to module map e.g spree_:Spree"
+
+    method_option :modules, :aliases => '-m', type: :array, :default => [],
+                  desc: "List of Modules to search for namespaced models"
+
+    def db()
+
+      start_connections
+
+      unless File.directory?(options[:path])
+        puts "WARNING : No such PATH found #{options[:path]} - trying mkdir"
+        FileUtils::mkdir_p(options[:path])
+      end
+
+      generator = options[:csv] ?  DataShift::CsvGenerator.new :  DataShift::ExcelGenerator.new
+
+      DataShift::Exporters::Configuration.from_hash(options)
+
+      ext = options[:csv] ? '.csv' : '.xls'
+
+      modules = [nil] + options[:modules]
+
+      ActiveRecord::Base.connection.tables.each do |table|
+
+        modules.each do |m|
+          @klass = DataShift::MapperUtils.table_to_arclass(table, m)
+          break if(@klass)
+        end
+
+        options[:prefix_map].each do |p, m|
+          @klass = DataShift::MapperUtils.table_to_arclass(table.gsub(p, ''), m)
+          break if(@klass)
+        end unless(@klass)
+
+        if(@klass.nil?)
+          puts  "ERROR: No Model found for Table [#{table}] - perhaps a prefix map required?"
+          next
+        end
+
+        result = File.join(options[:path], "#{table}#{ext}")
+
+        puts "Datashift: Start export to #{result} for [#{table}]"
+        begin
+          if(options[:associations])
+            logger.info("Datashift: Generating with associations")
+            generator.generate_with_associations(result, @klass)
+          else
+            generator.generate(result, @klass)
+          end
+        rescue => e
+          puts e
+          puts e.backtrace
+          puts "Warning: Error during export, data may be incomplete"
+        end
+      end
+
+    end
+
+    no_commands do
+
+      def start_connections
+
+        if File.exist?(File.expand_path('config/environment.rb'))
+          begin
+            require File.expand_path('config/environment.rb')
+          rescue => e
+            logger.error("Failed to initialise ActiveRecord : #{e.message}")
+            raise ConnectionError.new("Failed to initialise ActiveRecord : #{e.message}")
+          end
+
+        else
+          raise PathError.new('No config/environment.rb found - cannot initialise ActiveRecord')
+        end
+      end
+
+      def generate(generater)
+        model = options[:model]
+        result = options[:result]
+
+        DataShift::Exporters::Configuration.from_hash(options)
+
+        logger.info "Datashift: Starting template generation for #{generater.class.name} to #{result}"
+
+        klass = DataShift::MapperUtils::class_from_string(model)  #Kernel.const_get(model)
+
+        raise "ERROR: No such Model [#{model}] found - check valid model supplied via -model <Class>" if(klass.nil?)
+
+        begin
+
+          if(options[:associations])
+            logger.info("Datashift: Generating template including associations")
+            generater.generate_with_associations(result, klass)
+          else
+            generater.generate(result, klass)
+          end
+        rescue => e
+          puts e
+          puts e.backtrace
+          puts "Warning: Error during export, data may be incomplete"
+        end
+
+      end
+
+    end   # no_commands
+
+  end
+end