datashift 0.16.0 → 0.40.0

data/lib/datashift/transformation/remove.rb

@@ -0,0 +1,44 @@
+# Copyright:: (c) Autotelik Media Ltd 2016
+# Author ::   Tom Statter
+# Date ::     Feb 2016
+# License::   MIT
+#
+# Details::   Class to manage the removal of unwanted columns or data
+#
+
+module DataShift
+
+  module Transformation
+
+    class Remove
+
+      def remove_list
+        @remove_list ||= DataShift::Configuration.call.prep_remove_list
+      end
+
+      def association?(mm)
+        return false unless(mm.association_type?)
+        DataShift::Configuration.call.exclude_associations.include?(mm.operator)
+      end
+
+      # Specify columns to remove via DataShift::Exporters::Configuration
+      #
+      def unwanted_columns( columns )
+        columns.delete_if { |r| remove_list.include?( r.to_sym ) } unless remove_list.empty?
+      end
+
+      def unwanted_headers( headers )
+        headers.delete_if { |r| remove_list.include?( r.source.to_sym ) } unless remove_list.empty?
+      end
+
+      # Specify columns to remove via DataShift::Exporters::Configuration
+      #
+      def unwanted_model_methods( model_methods )
+        model_methods.delete_if { |r| remove_list.include?( r.operator.to_sym ) } unless remove_list.empty?
+      end
+
+    end
+
+  end
+
+end

data/lib/datashift/version.rb

@@ -0,0 +1,3 @@
+module DataShift
+  VERSION = '0.40.0'.freeze
+end

data/lib/exporters/configuration.rb

@@ -0,0 +1,84 @@
+# Copyright:: (c) Autotelik Media Ltd 2015
+# Author ::   Tom Statter
+# Date ::     Feb 2015
+# License::   MIT
+#
+# Details::   Read mappings and provide cache type services for source=>destination mappings
+#
+require 'erubis'
+
+module DataShift
+
+  module Exporters
+
+    class Configuration < DataShift::Configuration
+
+      # @param [Boolean] Stop processing and abort if a row fails to export
+      # Default is false
+      # @return [Boolean]
+      #
+      attr_accessor :abort_on_failure
+
+      # @param [Char] Char to use as the column delimter for csv format
+      # @return [Char]
+      #
+      attr_accessor :csv_delimiter
+
+      # @param [Boolean] Export association data in single column in JSON format
+      # @return [Boolean]
+      #
+      attr_accessor :json
+
+      # @param [String] Name for worksheet, otherwise uses Class name
+      # @return [String]
+      #
+      attr_accessor :sheet_name
+
+      def initialize
+        super
+        @abort_on_failure = false
+        @csv_delimiter = ','
+        @json = false
+        @sheet_name = ''
+      end
+
+      # @return [DataShift::Exporters::Configuration] DataShift's current configuration
+      def self.call
+        @configuration ||= Exporters::Configuration.new
+      end
+
+      def self.reset
+        @configuration = Exporters::Configuration.new
+      end
+
+      # Set DataShift's configure
+      # @param config [DataShift::Exporters::Configuration]
+      class << self
+        attr_writer :configuration
+      end
+
+      # Modify DataShift's current Export configuration
+      # ```
+      # DataShift::Exporters::Configuration.configure do |config|
+      #   config.abort_on_failure = false
+      # end
+      # ```
+      def self.configure
+        yield call
+      end
+
+      # Modify DataShift's current Export configuration from an options hash
+
+      def self.from_hash( options )
+
+        DataShift::Configuration.from_hash(options)
+
+        DataShift::Exporters::Configuration.configure do |config|
+          config.csv_delimiter = options[:csv_delimiter] if(options[:csv_delimiter])
+        end
+      end
+
+    end
+  end
+
+end

data/lib/exporters/csv_exporter.rb

@@ -1,13 +1,10 @@
-# Copyright:: (c) Autotelik Media Ltd 2011
+# Copyright:: (c) Autotelik Media Ltd 2015
 # Author ::   Tom Statter
-# Date ::     Aug 2011
 # License::   MIT
 #
 # Details::   Export a model to CSV
 #
 #
-require 'exporter_base'
-require 'csv_file'
 
 module DataShift
 
@@ -16,92 +13,97 @@ module DataShift
     include DataShift::Logging
     include DataShift::ColumnPacker
 
-    def initialize(filename)
-      super(filename)
+    def initialize
+      super
+
+      @csv_delimiter = ','
     end
 
     # Create CSV file from set of ActiveRecord objects
+    #
     # Options :
-    # => :filename
-    # => :text_delim => Char to use to delim columns, useful when data contain embedded ','
-    # => ::methods => List of methods to additionally call on each record
     #
-    def export(export_records, options = {})
+    # :csv_delim => Char to use to delim columns, useful when data contain embedded ','
+    #
+    def export(file_name, export_records, options = {})
+
+      @file_name = file_name
 
       records = [*export_records]
 
-      unless(records && records.size > 0)
-        logger.warn("No objects supplied for export")
+      unless records && !records.empty?
+        logger.warn('No objects supplied for export')
         return
       end
 
       first = records[0]
 
-      raise ArgumentError.new('Please supply set of ActiveRecord objects to export') unless(first.is_a?(ActiveRecord::Base))
+      raise ArgumentError.new('Please supply set of ActiveRecord objects to export') unless first.is_a?(ActiveRecord::Base)
+
+      @csv_delimiter = options[:csv_delim] if options[:csv_delim]
 
-      Delimiters.text_delim = options[:text_delim] if(options[:text_delim])
+      headers = Headers.klass_to_headers(first.class)
 
-      CSV.open( (options[:filename] || filename), "w" ) do |csv|
+      logger.debug "Writing out CSV Export. Columns delimited by [#{csv_delimiter}]"
 
-        csv.ar_to_headers( records )
+      remove_list = DataShift::Transformation::Remove.new.remove_list
+
+      CSV.open(file_name, 'w', col_sep: csv_delimiter ) do |csv|
+        csv << headers.sources
 
         records.each do |r|
-          next unless(r.is_a?(ActiveRecord::Base))
-          csv.ar_to_csv(r, options)
+          next unless r.is_a?(ActiveRecord::Base)
+          csv.ar_to_row(r, remove_list)
         end
       end
+
+      logger.info "CSV export completed for #{records.size} records"
     end
 
-    # Create an Excel file from list of ActiveRecord objects
-    # Specify which associations to export via :with or :exclude
-    # Possible values are : [:assignment, :belongs_to, :has_one, :has_many]
+    # Create CSV file from list of ActiveRecord objects
+    #
+    # Options :
     #
-    def export_with_associations(klass, records, options = {})
+    # :csv_delim => Char to use to delim columns, useful when data contain embedded ','
+    #
+    def export_with_associations(file_name, klass, records, options = {})
 
-      Delimiters.text_delim = options[:text_delim] if(options[:text_delim])
+      state = DataShift::Configuration.call.with
 
-      MethodDictionary.find_operators( klass )
+      DataShift::Configuration.call.with = :all
 
-      MethodDictionary.build_method_details( klass )
+      @file_name = file_name
 
-      only  = options[:only] ? [*options[:only]] : nil
+      @csv_delimiter = options[:csv_delim] if(options[:csv_delim])
 
-      # For each type belongs has_one, has_many etc find the operators
-      # and create headers, then for each record call those operators
-      operators = options[:with] || MethodDetail::supported_types_enum
+      headers = Headers.klass_to_headers(klass)
 
-      CSV.open( (options[:filename] || filename), "w" ) do |csv|
+      schema = DataFlowSchema.new
 
-        csv.ar_to_headers( records, operators, options)
+      model_methods = schema.klass_to_model_methods( klass )
 
-        details_mgr = MethodDictionary.method_details_mgrs[klass]
+      logger.debug "Writing out CSV Export for #{klass} with Associations. Columns delimited by [#{csv_delimiter}]"
 
-        records.each do |obj|
+      CSV.open(file_name, 'w', col_sep: csv_delimiter ) do |csv|
+        csv << headers.sources
 
+        records.each do |record|
           row = []
 
-          operators.each do |op_type|
-
-            operators_for_type = details_mgr.get_list(op_type)
-
-            next if(operators_for_type.nil? || operators_for_type.empty?)
-
-            operators_for_type.each do |md|
-
-              next if(only && !only.include?( md.name.to_sym ) )
-
-              if(MethodDetail.is_association_type?(op_type))
-                row << record_to_column( obj.send( md.operator ))    # pack association into single column
-              else
-                row << escape_for_csv( obj.send( md.operator ) )
-              end
-            end
+          model_methods.each do |model_method|
+            row << if model_method.association_type?
+                     record_to_column( record.send(model_method.operator) )
+                   else
+                     escape_for_csv( record.send(model_method.operator) )
+                   end
           end
-
-          csv << row # next record
+          csv.add_row(row)
         end
       end
 
-    end
+    ensure
+      DataShift::Configuration.call.with = state
+    end # end write file
+
   end
 end

data/lib/exporters/excel_exporter.rb

@@ -1,130 +1,149 @@
-# Copyright:: (c) Autotelik Media Ltd 2011
+# Copyright:: (c) Autotelik Media Ltd 2016
 # Author ::   Tom Statter
-# Date ::     Aug 2011
+# Date ::     Mar 2016
 # License::   MIT
 #
 # Details::   Export a model to Excel '97(-2007) file format.
 #
-# TOD : Can we switch between .xls and XSSF (POI implementation of Excel 2007 OOXML (.xlsx) file format.)
+# TODO: Can we switch between .xls and XSSF (POI implementation of Excel 2007 OOXML (.xlsx) file format.)
 #
 #
 module DataShift
 
-  require 'exporter_base'
-
-  require 'excel'
-
   class ExcelExporter < ExporterBase
 
     include DataShift::Logging
     include DataShift::ColumnPacker
 
-    def initialize(filename)
-      @filename = filename
+    include DataShift::ExcelBase
+
+    # Optional, otherwise  uses the standard collection of Model Methods for supplied klass
+    attr_accessor :data_flow_schema
+
+    def initialize
+      super
+
+      @data_flow_schema = nil
     end
 
     # Create an Excel file from list of ActiveRecord objects
-    def export(export_records, options = {})
+    def export(file_name, export_records, options = {})
+
+      @file_name = file_name
 
       records = [*export_records]
 
-      unless(records && records.size > 0)
-        logger.warn("No objects supplied for export")
+      if(records.nil? || records.empty?)
+        logger.warn('Excel Export - No objects supplied for export - no file written')
         return
       end
 
       first = records[0]
 
-      logger.info("Exporting #{records.size} #{first.class} to Excel")
+      raise(ArgumentError, 'Please supply set of ActiveRecord objects to export') unless first.is_a?(ActiveRecord::Base)
 
-      raise ArgumentError.new('Please supply set of ActiveRecord objects to export') unless(first.is_a?(ActiveRecord::Base))
+      klass = first.class
 
+      excel = start_excel(klass, options)
 
-      raise ArgumentError.new('Please supply array of records to export') unless records.is_a? Array
+      prepare_data_flow_schema(klass)
 
-      excel = Excel.new
+      export_headers(klass)
 
-      if(options[:sheet_name] )
-        excel.create_worksheet( :name => options[:sheet_name] )
-      else
-        excel.create_worksheet( :name => records.first.class.name )
-      end
-
-      excel.ar_to_headers(records)
+      logger.info("Exporting #{records.size} #{klass} to Excel")
 
       excel.ar_to_xls(records)
 
-      excel.write( filename() )
+      logger.info("Writing Excel to file [#{file_name}]")
+
+      excel.write( file_name )
+    end
+
+    def export_headers(klass)
+
+      headers = if(data_flow_schema)
+                  data_flow_schema.sources
+                else
+                  Headers.klass_to_headers(klass)
+                end
+
+      excel.set_headers( headers )
+
+      logger.info("Wrote headers for #{klass} to Excel")
+      headers
+    end
+
+    def prepare_data_flow_schema(klass)
+      @data_flow_schema = DataShift::DataFlowSchema.new
+      @data_flow_schema.prepare_from_klass( klass )
+
+      data_flow_schema
     end
 
     # Create an Excel file from list of ActiveRecord objects, includes relationships
     #
-    #   Options
+    # The Associations/relationships to include are driven by Configuration Options
     #
-    #     only          Specify (as symbols) columns (assignments) to export from klass
-    #     with:         Specify which association types to export :with
-    #                   Possible values are : [:assignment, :belongs_to, :has_one, :has_many]
-    #     with_only     Specify (as symbols) columns for association types to export
-    #     sheet_name    Else uses Class name
-    #     json:         Export association data in single column in JSON format
+    #   See - lib/exporters/configuration.rb
     #
-    def export_with_associations(klass, records, options = {})
-
-      records = [*records]
+    def export_with_associations(file_name, klass, records, options = {})
 
-      only  = options[:only] ? [*options[:only]] : nil
+      state = DataShift::Configuration.call.with
 
-      excel = Excel.new
+      DataShift::Configuration.call.with = :all
 
-      if(options[:sheet_name] )
-        excel.create_worksheet( :name => options[:sheet_name] )
-      else
-        excel.create_worksheet( :name => records.first.class.name )
-      end
+      @file_name = file_name
 
-      MethodDictionary.find_operators( klass )
+      excel = start_excel(klass, options)
 
-      MethodDictionary.build_method_details( klass )
+      logger.info("Processing [#{records.size}] #{klass} records to Excel")
 
-      # For each type belongs has_one, has_many etc find the operators
-      # and create headers, then for each record call those operators
-      operators = options[:with] || MethodDetail::supported_types_enum
+      prepare_data_flow_schema(klass)
 
-      excel.ar_to_headers( records, operators, options)
+      export_headers(klass)
 
-      details_mgr = MethodDictionary.method_details_mgrs[klass]
+      nodes = data_flow_schema.nodes
 
       row = 1
 
       records.each do |obj|
-
         column = 0
 
-        [*operators].each do |op_type|     # belongs_to, has_one, has_many etc
-
-          operators_for_type = details_mgr.get_list(op_type)
+        nodes.each do |node|
 
-          next if(operators_for_type.nil? || operators_for_type.empty?)
+          logger.info("Send to Excel: #{node.inspect}")
 
-          operators_for_type.each do |md|     # actual associations on obj
+          model_method = node.model_method
 
-            next if(only && !only.include?( md.name.to_sym ) )
+          logger.info("Send to Excel: #{model_method.pp}")
 
-            if(MethodDetail.is_association_type?(op_type))
-              excel[row, column] = record_to_column( obj.send( md.operator ), options )    # pack association into single column
+          begin
+            # pack association instances into single column
+            if model_method.association_type?
+              logger.info("Processing #{model_method.inspect} associations")
+              excel[row, column] = record_to_column( obj.send( model_method.operator ), configuration.json )
             else
-              excel[row, column] = obj.send( md.operator )
+              excel[row, column] = obj.send( model_method.operator )
             end
-            column += 1
+          rescue => x
+            logger.error("Failed to write #{model_method.inspect} to Excel")
+            logger.error(x.inspect)
           end
+
+          column += 1
         end
 
         row += 1
       end
 
-      excel.write( filename() )
+      logger.info("Writing Excel to file [#{file_name}]")
+      excel.write( file_name )
+
+    ensure
+      DataShift::Configuration.call.with = state
 
     end
+
   end # ExcelGenerator
 
-end # DataShift
+end # DataShift