datashift 0.16.0 → 0.40.0

data/lib/datashift/excel_base.rb

@@ -1,123 +1,135 @@
-# To change this template, choose Tools | Templates
-# and open the template in the editor.
+module DataShift
 
-module ExcelBase
-
-
-
-  def sanitize_sheet_name( name )
-    name.gsub(/[\[\]:\*\/\\\?]/, '')
-  end
+  module ExcelBase
 
+    def self.max_columns
+      1024
+    end
 
-  # TODO -revisit/refactor - maybe this should just be on the base class Excel
-  # so you can call direct like excel.parse_headers(options[:header_row])
-  # rather than
-  #   sheet = excel.worksheet( sheet_number )
-  #   parse_headers(sheet, options[:header_row])
+    attr_accessor :excel, :sheet
 
-  attr_accessor :header_row_index, :excel_headers
+    #  Create @excel and set @sheet
+    #
+    # Options  :
+    #
+    #     :sheet_name : Create a new worksheet assign to @sheet.
+    #                   Default is class.name
+    #
+    def start_excel(klass, options = {})
 
-  def parse_headers( sheet,  header_row = 0 )
+      @excel = DataShift::Excel.new
 
-    @header_row_index = header_row || 0
+      name = options[:sheet_name] ? options[:sheet_name] : klass.name
 
-    header_row = sheet.row(header_row_index)
+      @sheet = excel.create_worksheet( name: name )
 
-    raise MissingHeadersError, "No headers found - Check Sheet #{sheet} is complete and Row #{header_row_index} contains headers" unless(header_row)
+      unless sheet
+        logger.error("Excel failed to create WorkSheet called [#{name}]")
 
-    @excel_headers = []
+        raise "Failed to create Excel WorkSheet called [#{name}]"
+      end
 
-    # TODO - make more robust - currently end on first empty column
-    # There is no actual max columns in Excel .. you will run out of memory though at some point
-    (0..1024).each do |column|
-      cell = header_row[column]
-      break unless cell
-      header = "#{cell.to_s}".strip
-      break if header.empty?
-      @excel_headers << header
+      @excel
     end
 
-    @excel_headers
-  end
+    # Open excel file and assign to @excel and set @sheet
+    #
+    # Options  :
+    #
+    #     :sheet_name : Create a new worksheet assign to @sheet. Default is class.name
+    #
+    #     :sheet_number : Select the sheet by index - sheet_name takes precedence
+    #
+    def open_excel( file_name, options = {})
 
-  # Helpers for dealing with Active Record models and collections
-  # Specify array of operators/associations to include - possible values are :
-  #     [:assignment, :belongs_to, :has_one, :has_many]
+      @excel = DataShift::Excel.new
 
-  def ar_to_headers( records, associations = nil, options = {} )
-    return if( !records.first.is_a?(ActiveRecord::Base) || records.empty?)
+      @excel.open(file_name)
 
-    only = *options[:only] ? [*options[:only]] : nil
+      if options[:sheet_name]
 
-    headers =[]
+        @sheet = @excel.create_worksheet( name: options[:sheet_name] )
 
-    if associations
-      details_mgr = DataShift::MethodDictionary.method_details_mgrs[records.first.class]
+        unless sheet
+          logger.error("Excel failed to create WorkSheet for #{name}")
 
-      [*associations].each do |a|
+          raise "Failed to create Excel WorkSheet for #{name}"
+        end
 
-        details_mgr.get_list(a).each do  |md|
+      elsif options[:sheet_number]
+        @sheet = @excel.worksheet( options[:sheet_number] )
+      else
+        @sheet = @excel.worksheets.first
+      end
 
-          next if(only && !only.include?( md.name.to_sym ) )
+      @excel
+    end
 
-          puts md.name.to_sym.inspect
+    def parse_headers( sheet, header_row_idx = 0 )
 
-          headers << "#{md.operator}"
+      headers = DataShift::Headers.new(:excel, header_row_idx)
 
-        end
-      end if(details_mgr)
+      header_row = sheet.row(header_row_idx)
+      unless header_row
+        raise MissingHeadersError,
+              "No headers found - Check Sheet #{sheet} is complete and Row #{header_row_idx} contains headers"
+      end
 
-    else
+      # TODO: - make more robust - currently end on first empty column
+      # There is no actual max columns in Excel .. you will run out of memory though at some point
+      (0..ExcelBase.max_columns).each do |column|
+        cell = header_row[column]
+        break unless cell
+        header = cell.to_s.strip
+        break if header.empty?
+        headers << header
+      end
 
-      headers = records.first.class.columns.collect( &:name )
+      headers
     end
 
-    set_headers( headers )
-  end
-
+    def sanitize_sheet_name( name )
+      name.gsub(/[\[\]:\*\/\\\?]/, '')
+    end
 
-  # Pass a set of AR records
-  def ar_to_xls(records, options = {})
-    return if( ! records.first.is_a?(ActiveRecord::Base) || records.empty?)
+    # Pass a set of AR records
+    def ar_to_xls(records, options = {})
+      return if !records.first.is_a?(ActiveRecord::Base) || records.empty?
 
-    row_index =
-        if(options[:no_headers])
-          0
-        else
-          ar_to_headers( records )
-          1
-        end
+      # assume headers present
+      row_index = (options[:start_row]) ? (options[:start_row]) : 1
 
-    records.each do |record|
-      ar_to_xls_row(row_index, 0, record)
+      records.each do |record|
+        ar_to_xls_row(row_index, 0, record)
 
-      row_index += 1
+        row_index += 1
+      end
     end
-  end
-
 
-  # Save data from an AR record to the current row, based on the record's columns [c1,c2,c3]
-  # Returns the number of the final column written to  
-  def ar_to_xls_row(row, start_column, record)
-    return unless( record.is_a?(ActiveRecord::Base))
-
-    column = start_column
-    record.class.columns.each do |connection_column|
-      ar_to_xls_cell(row, column, record, connection_column)
-      column += 1
+    # Save data from an AR record to the current row, based on the record's columns [c1,c2,c3]
+    # Returns the number of the final column written to
+    def ar_to_xls_row(row, start_column, record)
+      return unless record.is_a?(ActiveRecord::Base)
+
+      column = start_column
+      record.class.columns.each do |connection_column|
+        ar_to_xls_cell(row, column, record, connection_column)
+        column += 1
+      end
+      column
     end
-    column
-  end
 
-  def ar_to_xls_cell(row, column, record, connection_column)
-    begin
+    def ar_to_xls_cell(row, column, record, connection_column)
+
       datum = record.send(connection_column.name)
 
       self[row, column] = datum
     rescue => e
-      puts "Failed to export #{datum} from #{connection_column.inspect} to column #{column}"
-      puts e, e.backtrace
+      logger.error("Failed to export #{datum} from #{connection_column.inspect} to column #{column}")
+      logger.error( e.message )
+      logger.error(e.backtrace)
+
     end
   end
+
 end

data/lib/datashift/exceptions.rb

@@ -1,21 +1,21 @@
-# Copyright:: (c) Autotelik Media Ltd 2014 
+# Copyright:: (c) Autotelik Media Ltd 2014
 # Author ::   Tom Statter
 # Date ::     June 2014
 # License::   Free, Open Source.
 #
 
 module DataShift
-  
+
   class DataShiftException < StandardError
 
     include DataShift::Logging
-        
+
     def initialize( msg )
       super
-      logger.error( msg)
+      logger.error(msg)
     end
 
-    def self.generate name
+    def self.generate(name)
       new_class = Class.new(DataShiftException) do
         def initialize( msg )
           super( msg )
@@ -26,35 +26,37 @@ module DataShift
     end
   end
 
-  class NilDataSuppliedError  < DataShiftException
-    def initialize( msg )
-      super( msg )
-    end
-  end
-  
-  class BadRuby < StandardError; end
-  
-  class UnsupportedFileType < StandardError; end
-  class BadFile < StandardError; end
-  
-  class MappingDefinitionError < StandardError; end
+  # Non self logging errors
 
+  class BadConfig < StandardError; end
+  class BadFile < StandardError; end
+  class BadRuby < StandardError; end
+  class BadUri < StandardError; end
 
+  class MappingDefinitionError < StandardError; end
   class MissingHeadersError < StandardError; end
   class MissingMandatoryError < StandardError; end
 
   class PathError < StandardError; end
-  
-  class BadUri < StandardError; end
 
+  class RuntimeError < StandardError; end
+
+  class UnsupportedFileType < StandardError; end
 end
 
-DataShift::DataShiftException.generate( "ConnectionError")
-DataShift::DataShiftException.generate( "CouldNotAssignAssociation")
-DataShift::DataShiftException.generate( "CreateAttachmentFailed")
-DataShift::DataShiftException.generate( "DataProcessingError")
-DataShift::DataShiftException.generate( "FileNotFound")
-DataShift::DataShiftException.generate( "NoSuchClassError")
-DataShift::DataShiftException.generate( "MissingConfigOptionError")
-DataShift::DataShiftException.generate( "RecordNotFound")
-DataShift::DataShiftException.generate( "SaveError")
+# Self logging errors
+
+DataShift::DataShiftException.generate( 'BadOperatorType')
+DataShift::DataShiftException.generate( 'ConnectionError')
+DataShift::DataShiftException.generate( 'CouldNotAssignAssociation')
+DataShift::DataShiftException.generate( 'CouldNotDeriveAssociationClass')
+DataShift::DataShiftException.generate( 'CreateAttachmentFailed')
+DataShift::DataShiftException.generate( 'DataProcessingError')
+DataShift::DataShiftException.generate( 'FileNotFound')
+DataShift::DataShiftException.generate( 'NilDataSuppliedError')
+DataShift::DataShiftException.generate( 'NoSuchClassError')
+DataShift::DataShiftException.generate( 'NoSuchOperator')
+DataShift::DataShiftException.generate( 'MissingConfigOptionError')
+DataShift::DataShiftException.generate( 'RecordNotFound')
+DataShift::DataShiftException.generate( 'SaveError')
+DataShift::DataShiftException.generate( 'SourceIsNotAClass')

data/lib/datashift/file_definitions.rb

@@ -59,10 +59,10 @@ module FileDefinitions
   attr_accessor :current_line
 
   # Set the delimiter to use when splitting a line - can be either a String, or a Regexp
-  attr_writer   :field_delim
+  attr_writer :field_delim
 
-  def initialize( line  = nil )
-    @key   = String.new
+  def initialize( line = nil )
+    @key = ''
     parse(line) unless line.nil?
   end
 
@@ -72,10 +72,9 @@ module FileDefinitions
   end
 
   def self.subclasses
-    @subclasses ||=[]
+    @subclasses ||= []
   end
 
-
   # Return the field delimiter used when splitting a line
   def field_delim
     @field_delim || ','
@@ -84,14 +83,14 @@ module FileDefinitions
   # Parse each line of a file based on the field definition, yields self for each successive line
   #
   def each( file )
-    File::new(file).each_line do |line|
+    File.new(file).each_line do |line|
       parse( line )
       yield self
     end
   end
 
   def fields
-    @fields = self.class.field_definition.collect {|f| instance_variable_get "@#{f}" }
+    @fields = self.class.field_definition.collect { |f| instance_variable_get "@#{f}" }
     @fields
   end
 
@@ -128,16 +127,18 @@ module FileDefinitions
     def add_field(field, add_accessor = true)
       @field_definition ||= []
       @field_definition << field.to_s
-      attr_accessor field  if(add_accessor)
+      attr_accessor field  if add_accessor
     end
 
-
     # Helper to generate methods that return the complete list of fixed width fields
     # and associated ranges in this File definition, and parse a line.
     # e.g create_field_definition %w{ trade_id  drOrCr ccy costCentre postingDate amount }
     #
     def create_fixed_definition( field_range_map )
-      raise ArgumentError.new('Please supply hash to create_fixed_definition') unless field_range_map.is_a? Hash
+
+      unless field_range_map.is_a?(Hash)
+        raise ArgumentError, 'Please supply hash to create_fixed_definition'
+      end
 
       keys = field_range_map.keys.collect(&:to_s)
       string_map = Hash[*keys.zip(field_range_map.values).flatten]
@@ -172,10 +173,9 @@ module FileDefinitions
 
     # Create accessors for each field
     def create_field_attr_accessors
-      self.field_definition.each {|f| attr_accessor f}
+      field_definition.each { |f| attr_accessor f }
     end
 
-
     ###############################
     # PARSING + FILE MANIPULATION #
     ###############################
@@ -185,15 +185,13 @@ module FileDefinitions
       limit = options[:limit]
       count = 0
       lines = []
-      File::new(file).each_line do |line|
+      File.new(file).each_line do |line|
         break if limit && ((count += 1) > limit)
-        lines << self.new( line )
+        lines << new( line )
       end
       lines
     end
 
-
-
     # Split a file, whose field definition is represented by self,
     # into seperate streams, based on the values of one if it's fields.
     #
@@ -216,9 +214,13 @@ module FileDefinitions
       unless filtered.empty?
         log :info, "Writing seperate streams to #{path}"
 
-        filtered.each { |strm, objects| RecsBase::write( {"keys_#{field}_#{strm}.csv" => objects.collect(&:key).join("\n")}, path) } if(options.key?(:keys))
+        filtered.each do |strm, objects|
+          RecsBase.write( { "keys_#{field}_#{strm}.csv" => objects.collect(&:key).join("\n") }, path)
+        end if options.key?(:keys)
 
-        filtered.each { |strm, objects| RecsBase::write( {"#{field}_#{strm}.csv" => objects.collect(&:current_line).join("\n")}, path) }
+        filtered.each do |strm, objects|
+          RecsBase.write( { "#{field}_#{strm}.csv" => objects.collect(&:current_line).join("\n") }, path)
+        end
       end
     end
 
@@ -247,19 +249,19 @@ module FileDefinitions
 
       filtered = {}
 
-      if( self.new.respond_to?(field) )
+      if new.respond_to?(field)
 
         log :info, "Splitting on #{field}"
 
         File.open( file_name ) do |t|
           t.each do |line|
-            next unless(line && line.chomp!)
-            x = self.new(line)
+            next unless line && line.chomp!
+            x = new(line)
 
-            value = x.send( field.to_sym )    # the actual field value from the specified field column
+            value = x.send( field.to_sym ) # the actual field value from the specified field column
             next if value.nil?
 
-            if( regex.nil? || value.match(regex) )
+            if regex.nil? || value.match(regex)
               filtered[value] ? filtered[value] << x : filtered[value] = [x]
             end
           end
@@ -268,11 +270,11 @@ module FileDefinitions
         log :warn, "Field [#{field}] nor defined for file definition #{self.class.name}"
       end
 
-      if( options[:sort])
+      if options[:sort]
         filtered.values.each( &:sort )
         return filtered
       end
-      return filtered
+      filtered
     end
 
     # Open and parse a file, replacing a value in the specfied field.
@@ -291,16 +293,17 @@ module FileDefinitions
     #
     def file_set_field_by_map( file_name, fields, value_map, regex = nil )
 
-      lines, objects = [],[]
+      lines = []
+      objects = []
 
-      if fields.is_a?(Array)
-        attribs = fields
-      else
-        attribs = "#{fields}".split(',')
-      end
+      attribs = if fields.is_a?(Array)
+                  fields
+                else
+                  fields.to_s.split(',')
+                end
 
       attribs.collect! do |attrib|
-        raise ArgumentError.new("Field: #{attrib} is not a field on #{self.class.name}") unless self.new.respond_to?(attrib)
+        raise ArgumentError, "Field: #{attrib} is not a field on #{self.class.name}" unless new.respond_to?(attrib)
       end
 
       log :info, "#{self.class.name} - updating field(s) #{fields} in #{file_name}"
@@ -309,14 +312,16 @@ module FileDefinitions
         t.each do |line|
           if line.chomp.empty?
             lines << line
-            objects << self.new
+            objects << new
             next
           end
-          x = self.new(line)
+          x = new(line)
 
           attribs.each do |a|
             old_value = x.instance_variable_get( "@#{a}" )
-            x.instance_variable_set( "@#{a}", value_map[old_value] ) if value_map[old_value] || (regex && old_value.keys.detect {|k| k.match(regx) })
+            if value_map[old_value] || (regex && old_value.keys.detect { |k| k.match(regx) })
+              x.instance_variable_set( "@#{a}", value_map[old_value] )
+            end
           end
 
           objects << x
@@ -324,9 +329,9 @@ module FileDefinitions
         end
       end
 
-      return lines, objects
+      [lines, objects]
     end
-  end   # END class methods
+  end # END class methods
 
   # Open and parse a file, replacing a value in the specfied field.
   # Does not update the file itself. Does not write a new output file.
@@ -345,9 +350,9 @@ module FileDefinitions
   #
   def file_set_field( file_name, field, old_value, new_value, regex = nil )
 
-    map = {old_value => new_value}
+    map = { old_value => new_value }
 
-    return file_set_field_by_map(file_name, field, map, regex)
+    file_set_field_by_map(file_name, field, map, regex)
   end
 
 end