datashift 0.15.0 → 0.16.0

data/lib/generators/csv_generator.rb

@@ -12,22 +12,82 @@ module DataShift
 
   class CsvGenerator < GeneratorBase
 
+    include DataShift::Logging
+
     def initialize(filename)
       super(filename)
     end
 
     # Create CSV file representing supplied Model
-    
-    def generate(model, options = {})
 
-      @filename = options[:filename] if  options[:filename]
+    def generate(klass, options = {})
+      @filename = options[:filename] if options[:filename]
+
+      prep_remove_list(options)
+
+      MethodDictionary.find_operators( klass )
+      @headers = MethodDictionary.assignments[klass]
+
+      @headers.delete_if{|h| @remove_list.include?( h.to_sym ) }
+
+      logger.info("CSVGenerator saving generated template #{@filename}")
+
+      CSV.open(@filename, "w") do |csv|
+        csv << @headers
+      end
+    end
+
+    def generate_with_associations(klass, options = {})
+      @filename = options[:filename] if options[:filename]
+
+      MethodDictionary.find_operators( klass )
+      MethodDictionary.build_method_details( klass )
+
+      work_list = MethodDetail::supported_types_enum.to_a - [ *options[:exclude] ]
+
+      prep_remove_list(options)
+
+      @headers = []
+
+      details_mgr = MethodDictionary.method_details_mgrs[klass]
+
+      work_list.each do |assoc_type|
+        method_details_for_assoc_type = details_mgr.get_list_of_method_details(assoc_type)
+
+        next if(method_details_for_assoc_type.nil? || method_details_for_assoc_type.empty?)
+
+        method_details_for_assoc_type.each do |md|
+          comparable_association = md.operator.to_s.downcase.to_sym
+
+          i = remove_list.index { |r| r == comparable_association }
+
+          (i) ? remove_list.delete_at(i) : headers << "#{md.operator}"
+        end
+      end
+
+      logger.info("CSVGenerator saving generated with associations template #{@filename}")
+
+      CSV.open(@filename, "w") do |csv|
+        csv << @headers
+      end
     end
 
-  
-    # Create an Csv file representing supplied Model
+
+    # Create an CSV file representing supplied Model
 
     def export(items, options = {})
     end
 
+
+    private
+
+    # Take options and create a list of symbols to remove from headers
+    #
+    def prep_remove_list( options )
+      @remove_list = [ *options[:remove] ].compact.collect{|x| x.to_s.downcase.to_sym }
+
+      @remove_list += GeneratorBase::rails_columns if(options[:remove_rails])
+    end
+
   end
 end

data/lib/generators/generator_base.rb

@@ -10,17 +10,83 @@ module DataShift
   class GeneratorBase
 
     attr_accessor :filename, :headers, :remove_list
-  
+
     def initialize(filename)
       @filename = filename
       @headers = []
       @remove_list =[]
     end
-    
-    
+
     def self.rails_columns
       @rails_standard_columns ||= [:id, :created_at, :created_on, :updated_at, :updated_on]
     end
+
+
+    # Parse options and build collection of headers for a method_details_mgr wrapping a class
+    # based on association requirements,
+    #
+    # Default is to include *everything*
+    #
+    # * <tt>:exclude</tt> - Association TYPE(s) to exclude completely.
+    #
+    #     Possible association_type values are given by MethodDetail::supported_types_enum
+    #       ... [:assignment, :belongs_to, :has_one, :has_many]
+    #
+    # * <tt>:remove</tt> - Array of header names to remove
+    #
+    # Rails DB columns like id, created_at, updated_at are removed by default
+    #
+    # * <tt>:include_rails</tt> - Specify to keep Rails columns in mappings
+    #
+    def prepare_model_headers(method_details_mgr, options = {})
+
+      work_list = MethodDetail::supported_types_enum.to_a - [ *options[:exclude] ]
+
+      @headers = []
+
+      work_list.each do |assoc_type|
+        method_details_for_assoc_type = method_details_mgr.get_list_of_method_details(assoc_type)
+
+        next if(method_details_for_assoc_type.nil? || method_details_for_assoc_type.empty?)
+
+        method_details_for_assoc_type.each do |md|
+          #comparable_association = md.operator.to_s.downcase.to_sym
+          #i = remove_list.index { |r| r == comparable_association }
+          #(i) ? remove_list.delete_at(i) : @headers << "#{md.operator}"
+          @headers << md.operator
+        end
+      end
+
+      remove_headers(options)
+
+    end
+
+    # Parse options and remove  headers
+    # Specify columns to remove with :
+    #   options[:remove]
+    # Rails columns like id, created_at are removed by default,
+    #  to keep them in specify
+    #   options[:include_rails]
+    #
+    def remove_headers(options)
+      remove_list = prep_remove_list( options )
+
+      #TODO - more efficient way ?
+      headers.delete_if { |h| remove_list.include?( h.to_sym ) } unless(remove_list.empty?)
+    end
+
+
+    # Take options and create a list of symbols to remove from headers
+    # Rails columns like id, created_at etc are added to the remove list by default
+    # Specify :include_rails to keep them in
+    def prep_remove_list( options )
+      remove_list = [ *options[:remove] ].compact.collect{|x| x.to_s.downcase.to_sym }
+
+      remove_list += GeneratorBase::rails_columns unless(options[:include_rails])
+
+      remove_list
+    end
+
   end
 
 end

data/lib/generators/mapping_generator.rb

@@ -0,0 +1,112 @@
+# Copyright:: (c) Autotelik Media Ltd 2015
+# Author ::   Tom Statter
+# Date ::     Aug 2015
+# License::   MIT
+#
+# Details::   Create mappings between systems
+#
+require 'generator_base'
+
+module DataShift
+
+  class MappingGenerator < GeneratorBase
+
+    include DataShift::Logging
+    include ExcelBase
+
+    def initialize(filename)
+      super(filename)
+    end
+
+    # Create an YAML template for mapping headers
+    #
+    # Options:
+    #
+    # * <tt>:model_as_dest</tt> - Override default treatment of using model as the SOURCE
+    #
+    # * <tt>:remove</tt> - Array of header names to remove
+    #
+    # Rails columns like id, created_at etc are added to the remove list by default
+    #
+    # * <tt>:include_rails</tt> - Specify to keep Rails columns in mappings
+    #
+    # * <tt>:associations</tt> - Additionally include all Associations
+    #
+    # * <tt>:exclude</tt> - Association TYPE(s) to exclude.
+    #
+    #     Possible association_type values are given by MethodDetail::supported_types_enum
+    #       ... [:assignment, :belongs_to, :has_one, :has_many]
+    #
+    # * <tt>:file</tt> - Write mappings direct to file name provided
+    #
+    def generate(model = nil, options = {})
+
+      mappings = "mappings:\n"
+
+      if(model)
+
+        klass = DataShift::ModelMapper.class_from_string_or_raise( model )
+
+        MethodDictionary.find_operators( klass )
+
+        MethodDictionary.build_method_details( klass )
+
+        prepare_model_headers(MethodDictionary.method_details_mgrs[klass], options)
+
+        if(options[:model_as_dest])
+          headers.each_with_index do |s, i|  mappings += "       #srcs_column_heading_#{i}: #{s}\n" end
+        else
+          headers.each_with_index do |s, i|  mappings += "       #{s}: #dest_column_heading_#{i}\n" end
+        end
+      else
+        mappings += <<EOS
+    ##source_column_heading_0: #dest_column_heading_0
+    ##source_column_heading_1: #dest_column_heading_1
+    ##source_column_heading_2: #dest_column_heading_2
+
+EOS
+      end
+
+      File.open(options[:file], 'w')  do |f| f << mappings  end if(options[:file])
+
+      mappings
+
+    end
+
+    # Create an YAML template from a Excel spreadsheet for mapping headers
+    #
+    # * <tt>:model_as_dest</tt> - Override default treatment of using model as the SOURCE
+    #
+    # * <tt>:file</tt> - Write mappings direct to file name provided
+    #
+    def generate_from_excel(excel_file_name, options = {})
+
+      excel = Excel.new
+
+      puts "\n\n\nGenerating mapping from Excel file: #{excel_file_name}"
+
+      excel.open(excel_file_name)
+
+      sheet_number = options[:sheet_number] || 0
+
+      sheet = excel.worksheet( sheet_number )
+
+      parse_headers(sheet, options[:header_row])
+
+      mappings = "mappings:\n"
+
+      if(options[:model_as_dest])
+        excel_headers.each_with_index do |s, i|  mappings += "       #srcs_column_heading_#{i}: #{s}\n" end
+      else
+        excel_headers.each_with_index do |s, i|  mappings += "       #{s}: #dest_column_heading_#{i}\n" end
+      end
+
+      File.open(options[:file], 'w')  do |f| f << mappings  end if(options[:file])
+
+      mappings
+
+    end
+
+  end
+
+end # DataShift

data/lib/helpers/core_ext/csv_file.rb

@@ -0,0 +1,33 @@
+# Copyright:: Autotelik Media Ltd
+# Author ::   Tom Statter
+# Date ::     July 2010
+# License::   
+#
+# Details::   Simple internal representation of Csv File
+
+require 'csv'
+
+class CSV
+
+  include DataShift::ColumnPacker
+
+  # Helpers for dealing with Active Record models and collections
+    # Specify array of operators/associations to include - possible values are :
+    #     [:assignment, :belongs_to, :has_one, :has_many]
+
+    def ar_to_headers( records, associations = nil, options = {} )
+      add_row( to_headers(records, associations, options) )
+    end
+
+    # Convert an AR instance to a set of CSV columns
+    # Additional non instance data can be included by supplying list of methods to call
+    # on the record
+    def ar_to_csv(record, options = {})
+      csv_data = record.serializable_hash.values.collect { |c| escape_for_csv(c) }
+
+      [*options[:methods]].each { |x| csv_data << escape_for_csv(record.send(x)) if(record.respond_to?(x)) } if(options[:methods])
+
+      add_row(csv_data)
+    end
+
+end

data/lib/loaders/csv_loader.rb

@@ -11,9 +11,9 @@ require 'datashift/exceptions'
 require 'datashift/method_mapper'
 
 module DataShift
-     
+
   module CsvLoading
-    
+
     include DataShift::Logging
 
     # Load data through active Record models into DB from a CSV file
@@ -31,9 +31,9 @@ module DataShift
     #   [:force_inclusion] : Array of inbound column names to force into mapping
     #   [:include_all]     : Include all headers in processing - takes precedence of :force_inclusion
     #   [:strict]          : Raise exception when no mapping found for a column heading (non mandatory)
-    
+
     def perform_csv_load(file_name, options = {})
-       
+
       require "csv"
 
       # TODO - can we abstract out what a 'parsed file' is - so a common object can represent excel,csv etc
@@ -48,19 +48,21 @@ module DataShift
       puts "\n\n\nLoading from CSV file: #{file_name}"
       puts "Processing #{@parsed_file.size} rows"
       begin
-  
+
         load_object_class.transaction do
           @reporter.reset
 
           @parsed_file.each_with_index do |row, i|
-            
-            @current_row = row 
-            
+
+            @current_row = row
+
             @reporter.processed_object_count += 1
-            
+
+            logger.info("Begin processing Row #{@reporter.processed_object_count}  from CSV file")
+
             begin
               # First assign any default values for columns not included in parsed_file
-              process_missing_columns_with_defaults
+              process_defaults
 
               # TODO - Smart sorting of column processing order ....
               # Does not currently ensure mandatory columns (for valid?) processed first but model needs saving
@@ -73,64 +75,64 @@ module DataShift
               # pulling data out of associated column
               @method_mapper.method_details.each_with_index do |method_detail, col|
 
+                unless method_detail
+                  logger.warn("No method_detail found for col #{col + 1} #{method_detail}")
+                  next # TODO populate unmapped with a real MethodDetail that is 'null' and create is_nil
+                end
+
                 value = row[col]
 
-                prepare_data(method_detail, value)
-            
-                process()
+                process(method_detail, value)
               end
 
             rescue => e
-              failure( row, true )
-              logger.error "Failed to process row [#{i}] (#{@current_row})"
-              
+              failure(row, true)
+              logger.error e.inspect
+              logger.error e.backtrace.first.inspect
+              logger.error "Failed to process row [#{@reporter.processed_object_count}] (#{@current_row})"
+
               if(verbose)
-                puts "Failed to process row [#{i}] (#{@current_row})" 
-                puts e.inspect 
+                puts "Failed to process row [#{@reporter.processed_object_count}] (#{@current_row})"
+                puts e.inspect
               end
-              
+
               # don't forget to reset the load object 
               new_load_object
               next
             end
-            
-            # TODO - make optional -  all or nothing or carry on and dump out the exception list at end        
-            unless(save)
-              failure
-              logger.error "Failed to save row [#{@current_row}] (#{load_object.inspect})"
-              logger.error load_object.errors.inspect if(load_object)
-            else
-              logger.info "Row #{@current_row} succesfully SAVED : ID #{load_object.id}"
-              @reporter.add_loaded_object(@load_object)
-            end
+
+            # TODO - make optional -  all or nothing or carry on and dump out the exception list at end
+
+            logger.debug "Attempting Save on : #{load_object.inspect}"
+
+            save_and_report
 
             # don't forget to reset the object or we'll update rather than create
             new_load_object
-
           end
-        
+
           raise ActiveRecord::Rollback if(options[:dummy]) # Don't actually create/upload to DB if we are doing dummy run
         end
       rescue => e
-        puts "CAUGHT ", e.backtrace, e.inspect
+        logger.error "perform_csv_load failed - #{e.message}:\n#{e.backtrace}"
         if e.is_a?(ActiveRecord::Rollback) && options[:dummy]
-          puts "CSV loading stage complete - Dummy run so Rolling Back."
+          logger.info "CSV loading stage complete - Dummy run so Rolling Back."
         else
           raise e
         end
       ensure
         report
-      end
-    
+      end     # transaction
+
     end
   end
-  
+
   class CsvLoader < LoaderBase
 
     include DataShift::CsvLoading
 
-    def initialize(klass, find_operators = true, object = nil, options = {})
-      super( klass, find_operators, object, options )
+    def initialize(klass, object = nil, options = {})
+      super( klass, object, options )
       raise "Cannot load - failed to create a #{klass}" unless @load_object
     end
 
@@ -141,5 +143,5 @@ module DataShift
     end
 
   end
-    
+
 end