datashift 0.16.0 → 0.40.0

data/lib/generators/generator_base.rb

@@ -1,92 +1,37 @@
-# Copyright:: (c) Autotelik Media Ltd 2011
+# Copyright:: (c) Autotelik Media Ltd 2015
 # Author ::   Tom Statter
-# Date ::     Aug 2010
 # License::   MIT
 #
-#  Details::  Base class for generators, which provide serivrs to describe a Model in an external format
+# Details::  Base class for generators, which provide services to describe a Model in an external format
 #
 module DataShift
 
   class GeneratorBase
 
-    attr_accessor :filename, :headers, :remove_list
+    include DataShift::Logging
 
-    def initialize(filename)
-      @filename = filename
-      @headers = []
-      @remove_list =[]
-    end
+    attr_accessor :configuration
 
-    def self.rails_columns
-      @rails_standard_columns ||= [:id, :created_at, :created_on, :updated_at, :updated_on]
+    def initialize
     end
 
-
-    # Parse options and build collection of headers for a method_details_mgr wrapping a class
-    # based on association requirements,
-    #
-    # Default is to include *everything*
-    #
-    # * <tt>:exclude</tt> - Association TYPE(s) to exclude completely.
-    #
-    #     Possible association_type values are given by MethodDetail::supported_types_enum
-    #       ... [:assignment, :belongs_to, :has_one, :has_many]
+    # Prepare to generate with associations but then
+    # calls a **derived generate** method i.e abstract to this base class
     #
-    # * <tt>:remove</tt> - Array of header names to remove
+    # file_name => Filename for generated template
     #
-    # Rails DB columns like id, created_at, updated_at are removed by default
-    #
-    # * <tt>:include_rails</tt> - Specify to keep Rails columns in mappings
-    #
-    def prepare_model_headers(method_details_mgr, options = {})
-
-      work_list = MethodDetail::supported_types_enum.to_a - [ *options[:exclude] ]
-
-      @headers = []
-
-      work_list.each do |assoc_type|
-        method_details_for_assoc_type = method_details_mgr.get_list_of_method_details(assoc_type)
-
-        next if(method_details_for_assoc_type.nil? || method_details_for_assoc_type.empty?)
-
-        method_details_for_assoc_type.each do |md|
-          #comparable_association = md.operator.to_s.downcase.to_sym
-          #i = remove_list.index { |r| r == comparable_association }
-          #(i) ? remove_list.delete_at(i) : @headers << "#{md.operator}"
-          @headers << md.operator
-        end
-      end
-
-      remove_headers(options)
-
-    end
-
-    # Parse options and remove  headers
-    # Specify columns to remove with :
-    #   options[:remove]
-    # Rails columns like id, created_at are removed by default,
-    #  to keep them in specify
-    #   options[:include_rails]
-    #
-    def remove_headers(options)
-      remove_list = prep_remove_list( options )
-
-      #TODO - more efficient way ?
-      headers.delete_if { |h| remove_list.include?( h.to_sym ) } unless(remove_list.empty?)
-    end
+    def generate_with_associations(file_name, klass)
 
+      state = DataShift::Configuration.call.with
 
-    # Take options and create a list of symbols to remove from headers
-    # Rails columns like id, created_at etc are added to the remove list by default
-    # Specify :include_rails to keep them in
-    def prep_remove_list( options )
-      remove_list = [ *options[:remove] ].compact.collect{|x| x.to_s.downcase.to_sym }
+      DataShift::Configuration.call.with = :all
 
-      remove_list += GeneratorBase::rails_columns unless(options[:include_rails])
+      generate(file_name, klass)
+    ensure
+      DataShift::Configuration.call.with = state
 
-      remove_list
     end
 
   end
 
-end
+end

data/lib/loaders/configuration.rb

@@ -0,0 +1,90 @@
+# Copyright:: (c) Autotelik Media Ltd 2015
+# Author ::   Tom Statter
+# Date ::     Feb 2015
+# License::   MIT
+#
+# Details::   Read mappings and provide cache type services for source=>destination mappings
+#
+require 'erubis'
+
+module DataShift
+
+  module Loaders
+
+    class Configuration < DataShift::Configuration
+
+      # Default is to stop processing once we hit a completely empty row. Over ride.
+      # WARNING maybe slow, as will process all rows as defined by Excel
+      # @param [Boolean]
+      #
+      attr_accessor :allow_empty_rows
+
+      # Destroy failed objects - if object.save fails at any point destroy the current object - all or nothing
+      # Default is true - database is cleaned up
+      # @param [Boolean]
+      #
+      attr_accessor :destroy_on_failure
+
+      # Stop processing and abort if any row fails to import
+      # Default is false - row reported as failure but loading continues
+      # @param [Boolean]
+      #
+      attr_accessor :abort_on_failure
+
+      # Row containing headers - default is 0
+      # @param [Integer]
+      #
+      attr_writer :header_row
+
+      def initialize
+        @mandatory = []
+        @allow_empty_rows = false
+        @abort_on_failure = false
+        @destroy_on_failure = true
+        @header_row = 0
+      end
+
+      # Custom Readers
+
+      def header_row
+        raise MissingHeadersError, "Minimum row for Headers is 0 - passed #{@header_row}" if @header_row.to_i < 0
+        @header_row
+      end
+
+      # @return [DataShift::Loaders::Configuration] DataShift's current configuration
+      def self.call
+        @configuration ||= Loaders::Configuration.new
+      end
+
+      def self.reset
+        @configuration = Loaders::Configuration.new
+      end
+
+      # Set DataShift's configure
+      # @param config [DataShift::Loaders::Configuration]
+      class << self
+        attr_writer :configuration
+      end
+
+      # Modify DataShift's current Import configuration
+      # ```
+      # DataShift::Loaders::Configuration.configure do |config|
+      #   config.verbose = false
+      # end
+      # ```
+      def self.configure
+        yield call
+      end
+
+      # Modify DataShift's current Import configuration from an options hash
+
+      def self.from_hash( options )
+        DataShift::Loaders::Configuration.configure do |config|
+          config.mandatory = options[:mandatory] if(options[:mandatory])
+        end
+      end
+
+    end
+  end
+
+end

data/lib/loaders/csv_loader.rb

@@ -4,144 +4,106 @@
 # License::   MIT
 #
 # Details::   Specific loader to support CSV files.
-# 
 #
-require 'loaders/loader_base'
-require 'datashift/exceptions'
-require 'datashift/method_mapper'
+#
+require_relative 'file_loader'
 
 module DataShift
 
-  module CsvLoading
+  class CsvLoader < LoaderBase
 
     include DataShift::Logging
+    include DataShift::FileLoader
+
+    def initialize
+      super
+    end
 
-    # Load data through active Record models into DB from a CSV file
-    #  
-    # Assumes header_row is first row i.e row 0
-    #   
-    #   
-    # OPTIONS :
-    #  
-    #  [:dummy]           : Perform a dummy run - attempt to load everything but then roll back
+    #  Options
+    #
+    #   [:allow_empty_rows]  : Default is to stop processing once we hit a completely empty row. Over ride.
+    #                          WARNING maybe slow, as will process all rows as defined by Excel
+    #
+    #   [:dummy]           : Perform a dummy run - attempt to load everything but then roll back
+    #
     #
-    #  Options passed through  to :  populate_method_mapper_from_headers
-    #  
-    #   [:mandatory]       : Array of mandatory column names
-    #   [:force_inclusion] : Array of inbound column names to force into mapping
-    #   [:include_all]     : Include all headers in processing - takes precedence of :force_inclusion
-    #   [:strict]          : Raise exception when no mapping found for a column heading (non mandatory)
+    def perform_load( _options = {} )
+      require 'csv'
 
-    def perform_csv_load(file_name, options = {})
+      raise "Cannot load - failed to create a #{klass}" unless load_object
 
-      require "csv"
+      logger.info "Starting bulk load from CSV : #{file_name}"
 
-      # TODO - can we abstract out what a 'parsed file' is - so a common object can represent excel,csv etc
+      # TODO: - can we abstract out what a 'parsed file' is - headers plus value of each node
+      # so a common object can represent excel,csv etc
       # then  we can make load() more generic
 
-      @parsed_file = CSV.read(file_name)
+      parsed_file = CSV.read(file_name)
 
-      # Create a method_mapper which maps list of headers into suitable calls on the Active Record class
-      # For example if model has an attribute 'price' will map columns called Price, price, PRICE etc to this attribute
-      populate_method_mapper_from_headers( @parsed_file.shift, options)
+      # assume headers are row 0
+      header_idx = 0
+      header_row = parsed_file.shift
+
+      set_headers( DataShift::Headers.new(:csv, header_idx, header_row) )
+
+      # maps list of headers into suitable calls on the Active Record class
+      bind_headers(headers)
 
-      puts "\n\n\nLoading from CSV file: #{file_name}"
-      puts "Processing #{@parsed_file.size} rows"
       begin
+        puts 'Dummy Run - Changes will be rolled back' if(configuration.dummy_run)
 
         load_object_class.transaction do
-          @reporter.reset
+          logger.info "Processing #{parsed_file.size} rows"
 
-          @parsed_file.each_with_index do |row, i|
+          parsed_file.each_with_index do |row, i|
+            current_row_idx = i
 
-            @current_row = row
+            logger.info "Processing Row #{i} : #{row}"
 
-            @reporter.processed_object_count += 1
+            # Iterate over the bindings, creating a context from data in associated Excel column
 
-            logger.info("Begin processing Row #{@reporter.processed_object_count}  from CSV file")
-
-            begin
-              # First assign any default values for columns not included in parsed_file
-              process_defaults
+            @binder.bindings.each_with_index do |method_binding, i|
+              unless method_binding.valid?
+                logger.warn("No binding was found for column (#{i})")
+                next
+              end
 
-              # TODO - Smart sorting of column processing order ....
-              # Does not currently ensure mandatory columns (for valid?) processed first but model needs saving
-              # before associations can be processed so user should ensure mandatory columns are prior to associations
+              value = row[method_binding.inbound_index] # binding contains column number
 
-              # as part of this we also attempt to save early, for example before assigning to
-              # has_and_belongs_to associations which require the load_object has an id for the join table
+              context = doc_context.create_node_context(method_binding, i, value)
 
-              # Iterate over the columns method_mapper found in Excel,
-              # pulling data out of associated column
-              @method_mapper.method_details.each_with_index do |method_detail, col|
+              logger.info "Processing Column #{method_binding.inbound_index} (#{method_binding.pp})"
 
-                unless method_detail
-                  logger.warn("No method_detail found for col #{col + 1} #{method_detail}")
-                  next # TODO populate unmapped with a real MethodDetail that is 'null' and create is_nil
+              begin
+                context.process
+              rescue => x
+                if doc_context.all_or_nothing?
+                  logger.error('All or nothing set and Current Columnfailed so complete Row aborted')
+                  break
                 end
-
-                value = row[col]
-
-                process(method_detail, value)
-              end
-
-            rescue => e
-              failure(row, true)
-              logger.error e.inspect
-              logger.error e.backtrace.first.inspect
-              logger.error "Failed to process row [#{@reporter.processed_object_count}] (#{@current_row})"
-
-              if(verbose)
-                puts "Failed to process row [#{@reporter.processed_object_count}] (#{@current_row})"
-                puts e.inspect
               end
+            end # end of each column(node)
 
-              # don't forget to reset the load object 
-              new_load_object
-              next
-            end
-
-            # TODO - make optional -  all or nothing or carry on and dump out the exception list at end
-
-            logger.debug "Attempting Save on : #{load_object.inspect}"
+            doc_context.save_and_monitor_progress
 
-            save_and_report
+            doc_context.reset unless doc_context.node_context.next_update?
+          end # all rows processed
 
-            # don't forget to reset the object or we'll update rather than create
-            new_load_object
+          if(configuration.dummy_run)
+            puts 'CSV loading stage done - Dummy run so Rolling Back.'
+            raise ActiveRecord::Rollback # Don't actually create/upload to DB if we are doing dummy run
           end
+        end # TRANSACTION N.B ActiveRecord::Rollback does not propagate outside of the containing transaction block
 
-          raise ActiveRecord::Rollback if(options[:dummy]) # Don't actually create/upload to DB if we are doing dummy run
-        end
       rescue => e
-        logger.error "perform_csv_load failed - #{e.message}:\n#{e.backtrace}"
-        if e.is_a?(ActiveRecord::Rollback) && options[:dummy]
-          logger.info "CSV loading stage complete - Dummy run so Rolling Back."
-        else
-          raise e
-        end
+        puts "ERROR: CSV loading failed : #{e.inspect}"
+        raise e
       ensure
         report
-      end     # transaction
-
-    end
-  end
+      end
 
-  class CsvLoader < LoaderBase
-
-    include DataShift::CsvLoading
-
-    def initialize(klass, object = nil, options = {})
-      super( klass, object, options )
-      raise "Cannot load - failed to create a #{klass}" unless @load_object
-    end
-
-    def perform_load( file_name, options = {} )
-      perform_csv_load( file_name, options )
-
-      puts "CSV loading stage complete - #{loaded_count} rows added."
+      puts 'CSV loading stage Complete.'
     end
 
   end
-
-end
+end

data/lib/loaders/excel_loader.rb

@@ -1,6 +1,5 @@
-# Copyright:: (c) Autotelik Media Ltd 2011
+# Copyright:: (c) Autotelik Media Ltd 2015
 # Author ::   Tom Statter
-# Date ::     Aug 2011
 # License::   MIT
 #
 # Details::   Specific loader to support Excel files.
@@ -10,127 +9,100 @@
 #             Iterates over all the rows using mapped operations to assign row data to a database object,
 #             i.e pulls data from each column and sends to object.
 #
-require 'datashift/exceptions'
-require 'datashift/exceptions'
+require_relative 'file_loader'
 
 module DataShift
 
-  require 'loaders/loader_base'
-
-  require 'excel'
-
-  module ExcelLoading
-
-    include ExcelBase
-
-    attr_accessor :excel
-
-    # Currently struggling to determine the 'end' of data in a spreadsheet
-    # this reflects if current row had any data at all
-    attr_reader :contains_data
-
-    def start_excel( file_name, options = {} )
-
-      @excel = Excel.new
-
-      excel.open(file_name)
-
-      puts "\n\n\nLoading from Excel file: #{file_name}"
-      logger.info("\nStarting Load from Excel file: #{file_name}")
-
-      sheet_number = options[:sheet_number] || 0
-
-      @sheet = excel.worksheet( sheet_number )
-
-      parse_headers(@sheet, options[:header_row])
-
-      raise MissingHeadersError, "No headers found - Check Sheet #{@sheet} is complete and Row #{header_row_index} contains headers" if(excel_headers.empty?)
+  class ExcelLoader < LoaderBase
 
-      # Create a method_mapper which maps list of headers into suitable calls on the Active Record class
-      # For example if model has an attribute 'price' will map columns called Price or price or PRICE etc to this attribute
-      populate_method_mapper_from_headers(excel_headers, options )
+    include DataShift::ExcelBase
+    include DataShift::FileLoader
 
-      reporter.reset
+    def initialize
+      super
     end
 
-    #  Options:
-    #   [:dummy]           : Perform a dummy run - attempt to load everything but then roll back
-    #  
+    #  Options
+    #
+    #   [:sheet_name]      : Create a new worksheet assign to @sheet. Default is class.name
     #   [:sheet_number]    : Default is 0. The index of the Excel Worksheet to use.
-    #   [:header_row]      : Default is 0. Use alternative row as header definition.
-    #   
-    #  Options passed through  to :  populate_method_mapper_from_headers
-    #  
-    #   [:mandatory]       : Array of mandatory column names
-    #   [:force_inclusion] : Array of inbound column names to force into mapping
-    #   [:include_all]     : Include all headers in processing - takes precedence of :force_inclusion
-    #   [:strict]          : Raise exception when no mapping found for a column heading (non mandatory)
+    #
+    def perform_load( options = {} )
 
-    def perform_excel_load( file_name, options = {} )
+      allow_empty_rows = DataShift::Loaders::Configuration.call.allow_empty_rows
+
+      logger.info "Starting bulk load from Excel : #{file_name}"
 
-      raise MissingHeadersError, "Minimum row for Headers is 0 - passed #{options[:header_row]}" if(options[:header_row] && options[:header_row].to_i < 0)
+      start(file_name, options)
 
-      start_excel(file_name, options)
+      # maps list of headers into suitable calls on the Active Record class
+      bind_headers(headers)
 
       begin
-        puts "Dummy Run - Changes will be rolled back" if options[:dummy]
+        puts 'Dummy Run - Changes will be rolled back' if(configuration.dummy_run)
 
         load_object_class.transaction do
-
-          @sheet.each_with_index do |row, i|
-
+          sheet.each_with_index do |row, i|
             current_row_idx = i
-            @current_row = row
 
-            next if(current_row_idx == header_row_index)
+            next if current_row_idx == headers.idx
 
             # Excel num_rows seems to return all 'visible' rows, which appears to be greater than the actual data rows
             # (TODO - write spec to process .xls with a huge number of rows)
             #
-            # This is rubbish but currently manually detect when actual data ends, this isn't very smart but
+            # manually have to detect when actual data ends, this isn't very smart but
             # got no better idea than ending once we hit the first completely empty row
-            break if(@current_row.nil? || @current_row.compact.empty?)
+            break if !allow_empty_rows && (row.nil? || row.empty?)
 
-            logger.info "Processing Row #{current_row_idx} : #{@current_row}"
+            logger.info "Processing Row #{current_row_idx}"
 
-            @contains_data = false
+            contains_data = false
 
-            begin
+            doc_context.progress_monitor.start_monitoring
 
-              process_excel_row(row)
+            # Iterate over the bindings,
+            # For each column bound to a model operator, create a context from data in associated Excel column
 
-              # This is rubbish but currently have to manually detect when actual data ends,
-              # no other way to detect when we hit the first completely empty row
-              break unless(contains_data == true)
+            @binder.bindings.each_with_index do |method_binding, _i|
+              unless method_binding.valid?
+                logger.warn("No binding was found for column (#{current_row_idx})")
+                next
+              end
 
-            rescue => e
-              process_excel_failure(e, true)
+              # get the value from the cell, binding contains the column number
+              value = row[method_binding.inbound_index]
 
-              # don't forget to reset the load object
-              new_load_object
-              next
-            end
+              context = doc_context.create_node_context(method_binding, current_row_idx, value)
 
-            break unless(contains_data == true)
+              contains_data ||= context.contains_data?
 
-            # currently here as we can only identify the end of a speadsheet by first empty row
-            @reporter.processed_object_count += 1
+              logger.info "Processing Column #{method_binding.inbound_index} (#{method_binding.pp})"
 
-            # TODO - make optional -  all or nothing or carry on and dump out the exception list at end
+              begin
+                context.process
+              rescue
+                if doc_context.all_or_nothing?
+                  logger.error('All or nothing set and Current Column failed so complete Row aborted')
+                  break
+                end
+              end
 
-            save_and_report
+            end
+
+            # Excel data rows not accurate, seems to have to manually detect when actual Excel data rows end
+            break if !allow_empty_rows && contains_data == false
 
-            # don't forget to reset the object or we'll update rather than create
-            new_load_object
+            doc_context.save_and_monitor_progress
 
-          end   # all rows processed
+            # unless next operation is update, reset the loader object
+            doc_context.reset unless doc_context.node_context.next_update?
+          end # all rows processed
 
-          if(options[:dummy])
-            puts "Excel loading stage complete - Dummy run so Rolling Back."
+          if(configuration.dummy_run)
+            puts 'Excel loading stage done - Dummy run so Rolling Back.'
             raise ActiveRecord::Rollback # Don't actually create/upload to DB if we are doing dummy run
           end
-
-        end   # TRANSACTION N.B ActiveRecord::Rollback does not propagate outside of the containing transaction block
+        end # TRANSACTION N.B ActiveRecord::Rollback does not propagate outside of the containing transaction block
 
       rescue => e
         puts "ERROR: Excel loading failed : #{e.inspect}"
@@ -139,86 +111,29 @@ module DataShift
         report
       end
 
+      puts 'Excel loading stage Complete.'
     end
 
-    def process_excel_failure( e, delete_object = true)
-      failure(@current_row, delete_object)
-
-      if(verbose)
-        puts "perform_excel_load failed in row [#{current_row_idx}] #{@current_row} - #{e.message} :"
-        puts e.backtrace
-      end
-
-      logger.error  "perform_excel_load failed in row [#{current_row_idx}] #{@current_row} - #{e.message} :"
-      logger.error e.backtrace.join("\n")
-    end
-
-
-    def value_at(row, column)
-      @excel[row, column]
-    end
-
-    def process_excel_row(row)
-
-      # First assign any default values for columns
-      process_defaults
-
-      # TODO - Smart sorting of column processing order ....
-      # Does not currently ensure mandatory columns (for valid?) processed first but model needs saving
-      # before associations can be processed so user should ensure mandatory columns are prior to associations
-
-      # as part of this we also attempt to save early, for example before assigning to
-      # has_and_belongs_to associations which require the load_object has an id for the join table
-
-      # Iterate over method_details, working on data out of associated Excel column
-      @method_mapper.method_details.each_with_index do |method_detail, i|
-
-        unless method_detail
-          logger.warn("No method_detail found for column (#{i})")
-          next # TODO populate unmapped with a real MethodDetail that is 'null' and create is_nil
-        end
-
-        logger.info "Processing Column #{method_detail.column_index} (#{method_detail.operator})"
-
-        value = row[method_detail.column_index]
-
-        @contains_data = true unless(value.nil? || value.to_s.empty?)
-
-        process(method_detail, value)
-      end
-
-    end
-
-  end
-
-
-  class ExcelLoader < LoaderBase
-
-    include ExcelLoading
+    private
 
-    # Setup loading
-    #
-    # Options to drive building the method dictionary for a class, enabling headers to be mapped to operators on that class.
+    #  Options  :
     #
-    # Options
-    #  :reload           : Force load of the method dictionary for object_class even if already loaded
-    #  :instance_methods : Include setter/delegate style instance methods for assignment, as well as AR columns
-    #  :verbose          : Verbose logging and to STDOUT
+    #   [:sheet_name]      : Create a new worksheet assign to @sheet. Default is class.name
+    #   [:sheet_number]    : Default is 0. The index of the Excel Worksheet to use.
     #
-    def initialize(klass, object = nil, options = {})
-      super( klass, object, options )
-      raise "Cannot load - failed to create a #{klass}" unless @load_object
-    end
+    def start( file_name, options = {} )
+      open_excel(file_name, options)
 
+      header_row = DataShift::Loaders::Configuration.call.header_row
 
-    def perform_load( file_name, options = {} )
+      set_headers( parse_headers(sheet, header_row) )
 
-      logger.info "Starting bulk load from Excel : #{file_name}"
-
-      perform_excel_load( file_name, options )
+      if headers.empty?
+        raise MissingHeadersError, "No headers found - Check Sheet #{sheet} is complete and Row #{headers.idx} contains headers"
+      end
 
-      puts "Excel loading stage complete - #{loaded_count} rows added."
+      excel
     end
 
   end
-end
+end