datashift 0.16.0 → 0.40.0

data/lib/datashift/thor_base.rb

@@ -1,38 +0,0 @@
-# Copyright:: (c) Autotelik Media Ltd 2012
-# Author ::   Tom Statter
-# Date ::     Dec 2014
-# License::   MIT.
-#
-# Note, not DataShift, case sensitive, create namespace for command line : datashift
-require 'thor'
-
-module DataShift
-
-  class DSThorBase < Thor
-
-    include DataShift::Logging
-
-    no_commands do
-
-      def start_connections()
-
-        # TODO - We're assuming run from a rails app/top level dir...
-
-        if(File.exists?(File.expand_path('config/environment.rb')))
-          begin
-            require File.expand_path('config/environment.rb')
-          rescue => e
-            logger.error("Failed to initialise ActiveRecord : #{e.message}")
-            raise ConnectionError.new("No config/environment.rb found - cannot initialise ActiveRecord")
-          end
-
-        else
-          raise PathError.new("No config/environment.rb found - cannot initialise ActiveRecord")
-          # TODO make this more robust ? e.g what about when using active record but not in Rails app, Sinatra etc
-        end
-      end
-    end
-
-  end
-
-end

data/lib/generators/mapping_generator.rb

@@ -1,112 +0,0 @@
-# Copyright:: (c) Autotelik Media Ltd 2015
-# Author ::   Tom Statter
-# Date ::     Aug 2015
-# License::   MIT
-#
-# Details::   Create mappings between systems
-#
-require 'generator_base'
-
-module DataShift
-
-  class MappingGenerator < GeneratorBase
-
-    include DataShift::Logging
-    include ExcelBase
-
-    def initialize(filename)
-      super(filename)
-    end
-
-    # Create an YAML template for mapping headers
-    #
-    # Options:
-    #
-    # * <tt>:model_as_dest</tt> - Override default treatment of using model as the SOURCE
-    #
-    # * <tt>:remove</tt> - Array of header names to remove
-    #
-    # Rails columns like id, created_at etc are added to the remove list by default
-    #
-    # * <tt>:include_rails</tt> - Specify to keep Rails columns in mappings
-    #
-    # * <tt>:associations</tt> - Additionally include all Associations
-    #
-    # * <tt>:exclude</tt> - Association TYPE(s) to exclude.
-    #
-    #     Possible association_type values are given by MethodDetail::supported_types_enum
-    #       ... [:assignment, :belongs_to, :has_one, :has_many]
-    #
-    # * <tt>:file</tt> - Write mappings direct to file name provided
-    #
-    def generate(model = nil, options = {})
-
-      mappings = "mappings:\n"
-
-      if(model)
-
-        klass = DataShift::ModelMapper.class_from_string_or_raise( model )
-
-        MethodDictionary.find_operators( klass )
-
-        MethodDictionary.build_method_details( klass )
-
-        prepare_model_headers(MethodDictionary.method_details_mgrs[klass], options)
-
-        if(options[:model_as_dest])
-          headers.each_with_index do |s, i|  mappings += "       #srcs_column_heading_#{i}: #{s}\n" end
-        else
-          headers.each_with_index do |s, i|  mappings += "       #{s}: #dest_column_heading_#{i}\n" end
-        end
-      else
-        mappings += <<EOS
-    ##source_column_heading_0: #dest_column_heading_0
-    ##source_column_heading_1: #dest_column_heading_1
-    ##source_column_heading_2: #dest_column_heading_2
-
-EOS
-      end
-
-      File.open(options[:file], 'w')  do |f| f << mappings  end if(options[:file])
-
-      mappings
-
-    end
-
-    # Create an YAML template from a Excel spreadsheet for mapping headers
-    #
-    # * <tt>:model_as_dest</tt> - Override default treatment of using model as the SOURCE
-    #
-    # * <tt>:file</tt> - Write mappings direct to file name provided
-    #
-    def generate_from_excel(excel_file_name, options = {})
-
-      excel = Excel.new
-
-      puts "\n\n\nGenerating mapping from Excel file: #{excel_file_name}"
-
-      excel.open(excel_file_name)
-
-      sheet_number = options[:sheet_number] || 0
-
-      sheet = excel.worksheet( sheet_number )
-
-      parse_headers(sheet, options[:header_row])
-
-      mappings = "mappings:\n"
-
-      if(options[:model_as_dest])
-        excel_headers.each_with_index do |s, i|  mappings += "       #srcs_column_heading_#{i}: #{s}\n" end
-      else
-        excel_headers.each_with_index do |s, i|  mappings += "       #{s}: #dest_column_heading_#{i}\n" end
-      end
-
-      File.open(options[:file], 'w')  do |f| f << mappings  end if(options[:file])
-
-      mappings
-
-    end
-
-  end
-
-end # DataShift

data/lib/helpers/core_ext/csv_file.rb

@@ -1,33 +0,0 @@
-# Copyright:: Autotelik Media Ltd
-# Author ::   Tom Statter
-# Date ::     July 2010
-# License::   
-#
-# Details::   Simple internal representation of Csv File
-
-require 'csv'
-
-class CSV
-
-  include DataShift::ColumnPacker
-
-  # Helpers for dealing with Active Record models and collections
-    # Specify array of operators/associations to include - possible values are :
-    #     [:assignment, :belongs_to, :has_one, :has_many]
-
-    def ar_to_headers( records, associations = nil, options = {} )
-      add_row( to_headers(records, associations, options) )
-    end
-
-    # Convert an AR instance to a set of CSV columns
-    # Additional non instance data can be included by supplying list of methods to call
-    # on the record
-    def ar_to_csv(record, options = {})
-      csv_data = record.serializable_hash.values.collect { |c| escape_for_csv(c) }
-
-      [*options[:methods]].each { |x| csv_data << escape_for_csv(record.send(x)) if(record.respond_to?(x)) } if(options[:methods])
-
-      add_row(csv_data)
-    end
-
-end

data/lib/helpers/core_ext/to_b.rb

@@ -1,24 +0,0 @@
-class Object
-  def to_b
-    case self
-    when true, false then self
-    when nil then false
-    else
-      to_i != 0
-    end
-  end
-end
-
-class String
-  TRUE_REGEXP = /^(yes|true|on|t|1|\-1)$/i.freeze
-  FALSE_REGEXP = /^(no|false|off|f|0)$/i.freeze
-
-  def to_b
-    case self
-    when TRUE_REGEXP then true
-    when FALSE_REGEXP then false
-    else
-      to_i != 0
-    end
-  end
-end

data/lib/loaders/reporter.rb

@@ -1,58 +0,0 @@
-# Copyright:: (c) Autotelik Media Ltd 2011
-# Author ::   Tom Statter
-# Date ::     Dec 2012
-# License::   MIT
-#
-# Details::   Store and report stats
-#
-module DataShift
-
-
-  class Reporter
-
-    include DataShift::Logging
-
-    # actual data rows/objects inbound
-    attr_accessor :processed_object_count
-    alias_method :processed_inbound_count, :processed_object_count
-
-    # DB objects created, updated etc
-    attr_accessor :loaded_objects, :failed_objects
-
-    # actual data rows/objects inbound
-    attr_accessor :success_inbound_count, :failed_inbound_count
-
-    def initialize()
-      reset
-    end
-
-    def reset()
-      @processed_object_count = 0
-      @loaded_objects, @failed_objects = [], []
-      @success_inbound_count, @failed_inbound_count = 0,0
-    end
-
-    def add_loaded_object(object)
-      @loaded_objects << object.id unless(object.nil? || @loaded_objects.include?(object))
-    end
-    
-    def add_failed_object(object)
-      @failed_objects << object unless( object.nil? || @failed_objects.include?(object))
-    end
-    
-    def report
-      loaded_objects.compact! if(loaded_objects)
-      
-      puts "\nProcessing Summary Report"
-      puts ">>>>>>>>>>>>>>>>>>>>>>>>>\n"
-      puts "Processed total of #{processed_object_count} inbound #{processed_object_count > 1 ? 'entries' : 'entry'}"
-      puts "#{loaded_objects.size}\tdatabase objects were successfully processed."
-      puts "#{success_inbound_count}\tinbound rows were successfully processed."
-
-      puts "There were NO failures." if failed_objects.empty?
-        
-      puts "WARNING : Check logs : #{failed_objects.size} rows contained errors" unless failed_objects.empty?
-    end
-    
-  end
-end

data/lib/thor/export.thor

@@ -1,175 +0,0 @@
-# Copyright:: (c) Autotelik Media Ltd 2012
-# Author ::   Tom Statter
-# Date ::     April 2012
-# License::   MIT.
-#
-# Usage::
-#
-#  To pull Datashift commands into your main application :
-#
-#     require 'datashift'
-#
-#     DataShift::load_commands
-#
-#  Cmd Line:
-#
-# => bundle exec thor datashift:export:excel -m <active record class> -r <output_template.xls> -a
-#
-require 'thor_base'
-
-
-require 'datashift'
-
-# Note, for thor not DataShift, case sensitive, want namespace for cmd line to be : datashift
-module Datashift
-
-          
-  class Export < DataShift::DSThorBase
-
-    desc "excel", "export any active record model (with optional associations)" 
-    method_option :model, :aliases => '-m', :required => true, :desc => "The active record model to export"
-    method_option :result, :aliases => '-r', :required => true, :desc => "Create template of model in supplied file"
-    method_option :assoc, :aliases => '-a', :type => :boolean, :desc => "Include all associations in the template"
-    method_option :exclude, :aliases => '-e',  :type => :array, :desc => "Use with -a : Exclude association types. Any from #{DataShift::MethodDetail::supported_types_enum.to_a.inspect}"
-    
-    def excel()
-
-      start_connections
-
-      require 'excel_exporter'
-
-      model = options[:model]
-      result = options[:result]
-     
-      logger.info "Datashift: Start Excel export to #{result}"
-            
-      klass = DataShift::ModelMapper::class_from_string(model)  #Kernel.const_get(model)
-    
-      raise "ERROR: No such Model [#{model}] found - check valid model supplied via -model <Class>" if(klass.nil?)
-
-      begin
-        gen = DataShift::ExcelExporter.new(result)
-
-        if(options[:assoc])
-          opts = (options[:exclude]) ? {:exclude => options[:exclude]} : {}
-          logger.info("Datashift: Exporting with associations")
-          gen.export_with_associations(klass, klass.all, opts)
-        else
-          gen.export(klass.all, :sheet_name => klass.name)
-        end
-      rescue => e
-        puts e
-        puts e.backtrace
-        puts "Warning: Error during export, data may be incomplete"
-      end
-
-    end
-  
-    desc "csv", "export any active record model (with optional associations)" 
-    method_option :model, :aliases => '-m', :required => true, :desc => "The active record model to export"
-    method_option :result, :aliases => '-r', :required => true, :desc => "Create template of model in supplied file"
-    method_option :assoc, :aliases => '-a', :type => :boolean, :desc => "Include all associations in the template"
-    method_option :exclude, :aliases => '-e',  :type => :array, :desc => "Use with -a : Exclude association types. Any from #{DataShift::MethodDetail::supported_types_enum.to_a.inspect}"
-    method_option :methods, :aliases => '-c',  :type => :array, :desc => "List of additional methods to call on model, useful for situations like delegated methods"
-    
-    def csv()
-     
-      # TODO - We're assuming run from a rails app/top level dir...
-      # ...can we make this more robust ? e.g what about when using active record but not in Rails app, 
-      require File.expand_path('config/environment.rb')
-   
-      require 'csv_exporter'
-
-      model = options[:model]
-      result = options[:result]
-     
-      logger.info "Datashift: Start CSV export to #{result}"
-            
-      klass = DataShift::ModelMapper::class_from_string(model)  #Kernel.const_get(model)
-    
-      raise "ERROR: No such Model [#{model}] found - check valid model supplied via -model <Class>" if(klass.nil?)
-
-      begin
-        gen = DataShift::CsvExporter.new(result)
-
-        if(options[:assoc])
-          logger.info("Datashift: Exporting with associations")
-          gen.export_with_associations(klass, klass.all, options)
-        else
-          gen.export(klass.all, options)
-        end
-      rescue => e
-        puts e
-        puts e.backtrace
-        puts "Warning: Error during export, data may be incomplete"
-      end
-
-    end
-    
-    desc "db", "Export every Active Record model" 
-
-    method_option :result, :aliases => '-r', :required => true, :desc => "Path in which to create excel files"
-    method_option :csv, :aliases => '-c', :desc => "Export to CSV instead - Excel is default."
-    method_option :prefix, :aliases => '-p', :desc => "For namespaced tables/models specify the table prefix e.g spree_"
-    method_option :module, :aliases => '-m', :desc => "For namespaced tables/models specify the Module name e.g Spree"
-    method_option :assoc, :aliases => '-a', :type => :boolean, :desc => "Include all associations in the template"
-    method_option :exclude, :aliases => '-e',  :type => :array, :desc => "Use with -a : Exclude association types. Any from #{DataShift::MethodDetail::supported_types_enum.to_a.inspect}"
- 
-    def db()
-     
-      require File.expand_path('config/environment.rb')
-      
-      require 'excel_exporter'
-      require 'csv_exporter'
-        
-      exporter = options[:csv] ?  DataShift::CsvExporter.new(nil) :  DataShift::ExcelExporter.new(nil)
-      
-      ext = options[:csv] ? '.csv' : '.xls'
-      
-      # Hmmm not many models appear - Rails uses autoload !
-      #ActiveRecord::Base.send(:subclasses).each do |model|
-       # puts model.name
-      #end
-       
-      parent = options[:module] ? Object.const_get(options[:module]) : Object
-      
-      ActiveRecord::Base.connection.tables.each do |table|
-            
-        table.sub!(options[:prefix],'') if(options[:prefix])
-
-        @result = File.join(options[:result], "#{table}#{ext}")
-        
-        begin      
-          @klass = parent.const_get(table.classify)    
-        rescue => e
-          puts e.inspect
-          puts "WARNING: Could not find an AR model for Table #{table}"
-          next
-        end
-
-        puts "Datashift: Start export to #{@result}"
-         
-        exporter.filename = @result
-        
-        raise "ERROR: No such Model [#{@klass}] found - check valid model supplied via -model <Class>" if(@klass.nil?)
-
-        begin
-          
-          if(options[:assoc])
-            opts = (options[:exclude]) ? {:exclude => options[:exclude]} : {}
-            logger.info("Datashift: Exporting with associations")
-            exporter.export_with_associations(@klass, @klass.all, opts)
-          else
-            exporter.export(@klass.all, :sheet_name => @klass.name)
-          end
-        rescue => e
-          puts e
-          puts e.backtrace
-          puts "Warning: Error during export, data may be incomplete"
-        end
-      end
-    end
-    
-  end
-
-end

data/lib/thor/generate.thor

@@ -1,191 +0,0 @@
-# Copyright:: (c) Autotelik Media Ltd 2012
-# Author ::   Tom Statter
-# Date ::     Mar 2012
-# License::   MIT.
-#
-# Usage::
-#
-#  To pull Datashift commands into your main application :
-#
-#     require 'datashift'
-#
-#     DataShift::load_commands
-#
-#  Cmd Line:
-#
-# => bundle exec thor list datashift
-#    bundle exec thor help  datashift:generate:excel
-#
-require 'datashift'
-require 'generators/excel_generator'
-
-# Note, not DataShift, case sensitive, create namespace for command line : datashift
-module Datashift
-
-          
-  class Generate < Thor     
-         
-    include DataShift::Logging
-      
-    desc "excel", "generate a template from an active record model (with optional associations)" 
-    method_option :model, :aliases => '-m', :required => true, :desc => "The active record model to export"
-    method_option :result, :aliases => '-r', :required => true, :desc => "Create template of model in supplied file"
-    method_option :assoc, :aliases => '-a', :type => :boolean, :desc => "Include all associations in the template"
-    method_option :exclude, :aliases => '-x',  :type => :array, :desc => "Use with -a : Exclude association types. Any from #{DataShift::MethodDetail::supported_types_enum.to_a.inspect}"
-    method_option :remove, :aliases => '-e',  :type => :array, :desc => "Don't generate in template the supplied fields"
-    method_option :remove_rails, :type => :boolean, :desc => "Don't generate in template the standard Rails fields: #{DataShift::GeneratorBase::rails_columns.inspect}"
-    
-    
-    def excel()
-     
-      # TODO - We're assuming run from a rails app/top level dir...
-      # ...can we make this more robust ? e.g what about when using active record but not in Rails app, 
-      require File.expand_path('config/environment.rb')
-
-      model = options[:model]
-      result = options[:result]
-     
-      logger.info "Datashift: Start Excel template generation in #{result}"
-
-      klass = DataShift::ModelMapper.class_from_string_or_raise( model )
-      
-      begin
-        gen = DataShift::ExcelGenerator.new(result)
-
-        opts =  { :remove => options[:remove],
-          :remove_rails => options[:remove_rails]
-        }
-          
-        if(options[:assoc])
-          
-          opts[:exclude] = options[:exclude]
-          
-          logger.info("Datashift: Generating with associations")
-          gen.generate_with_associations(klass, opts)
-        else
-          gen.generate(klass, opts)
-        end
-      rescue => e
-        puts e
-        puts e.backtrace
-        puts "Warning: Error during generation, template may be incomplete"
-      end
-
-    end
-    
-    
-    desc "csv", "generate a template from an active record model (with optional associations)" 
-    method_option :model, :aliases => '-m', :required => true, :desc => "The active record model to export"
-    method_option :result, :aliases => '-r', :required => true, :desc => "Create template of model in supplied file"
-    method_option :assoc, :aliases => '-a', :type => :boolean, :desc => "Include all associations in the template"
-    method_option :exclude, :aliases => '-x',  :type => :array, :desc => "Use with -a : Exclude association types. Any from #{DataShift::MethodDetail::supported_types_enum.to_a.inspect}"
-    
-    def csv()
-     
-      # TODO - We're assuming run from a rails app/top level dir...
-      # ...can we make this more robust ? e.g what about when using active record but not in Rails app, 
-      require File.expand_path('config/environment.rb')
-   
-      require 'csv_generator'
-
-      model = options[:model]
-      result = options[:result]
-     
-      logger.info "Datashift: Start CSV template generation in #{result}"
-            
-      begin
-        # support modules e.g "Spree::Property") 
-        klass = ModelMapper::class_from_string(model)  #Kernel.const_get(model)
-      rescue NameError => e
-        puts e
-        raise Thor::Error.new("ERROR: No such Model [#{model}] found - check valid model supplied")
-      end
-
-      raise Thor::Error.new("ERROR: No such Model [#{model}] found - check valid model supplied") unless(klass)
-      
-      begin
-        gen = DataShift::CsvGenerator.new(result)
-
-        if(options[:assoc])
-          opts = (options[:exclude]) ? {:exclude => options[:exclude]} : {}
-          logger.info("Datashift: Generating with associations")
-          gen.generate_with_associations(klass, opts)
-        else
-          gen.generate(klass)
-        end
-      rescue => e
-        puts e
-        puts e.backtrace
-        puts "Warning: Error during generation, template may be incomplete"
-      end
-
-    end
-    
-    desc "db", "Generate a template for every Active Record model" 
-
-    method_option :result, :aliases => '-r', :required => true, :desc => "Path in which to create excel files"
-    method_option :csv, :aliases => '-c', :desc => "Export to CSV instead - Excel is default."
-    method_option :prefix, :aliases => '-p', :desc => "For namespaced tables/models specify the table prefix e.g spree_"
-    method_option :module, :aliases => '-m', :desc => "For namespaced tables/models specify the Module name e.g Spree"
-    method_option :assoc, :aliases => '-a', :type => :boolean, :desc => "Include all associations in the template"
-    method_option :exclude, :aliases => '-x',  :type => :array, :desc => "Use with -a : Exclude association types. Any from #{DataShift::MethodDetail::supported_types_enum.to_a.inspect}"
-    method_option :remove, :aliases => '-e',  :type => :array, :desc => "Don't generate the user supplied fields"
-    method_option :remove_rails, :type => :boolean, :desc => "Don't generate the standard Rails fields: #{DataShift::GeneratorBase::rails_columns.inspect}"
-     
-    def db()
-     
-      require File.expand_path('config/environment.rb')
-      
-      require 'excel_exporter'
-      require 'csv_exporter'
-        
-      exporter = options[:csv] ?  DataShift::CsvGenerator.new(nil) :  DataShift::ExcelGenerator.new(nil)
-      
-      ext = options[:csv] ? '.csv' : '.xls'
-       
-      parent = options[:module] ? Object.const_get(options[:module]) : Object
-      
-      ActiveRecord::Base.connection.tables.each do |table|
-            
-        table.sub!(options[:prefix],'') if(options[:prefix])
-
-        @result = File.join(options[:result], "#{table}#{ext}")
-        
-        begin      
-          @klass = parent.const_get(table.classify)    
-        rescue => e
-          puts e.inspect
-          puts "WARNING: Could not find an AR model for Table #{table}"
-          next
-        end
-
-        puts "Datashift: Start template generation to #{@result}" 
-        
-        raise "ERROR: No such Model [#{@klass}] found - check valid model supplied via -model <Class>" if(@klass.nil?)
-
-        begin
-          opts =  { :filename => @result,
-                    :remove => options[:remove],
-                    :remove_rails => options[:remove_rails],
-                    :sheet_name => @klass.name
-          }
-          
-          if(options[:assoc])   
-            opts[:exclude] = options[:exclude]
-            logger.info("Datashift: Generating with associations")
-            exporter.generate_with_associations(@klass, opts)
-          else
-            exporter.generate(@klass, opts)
-          end
-        rescue => e
-          puts e
-          puts e.backtrace
-          puts "Warning: Error during export, data may be incomplete"
-        end
-      end
-    end
-    
-    
-  end
-
-end