datashift 0.0.1

data/lib/datashift/mapping_file_definitions.rb

@@ -0,0 +1,88 @@
+# This class provides a value map (hash) from a text mapping file
+#
+# The map file is  a text file of delimeted key -> values pairs
+#
+#  SUPPORTED FILE FORMATS:
+#
+#  2 column e.g. a,b
+#   creates a simple hash {a => b)
+#
+#  3 column e.g. a,b,c
+#   a,b becomes the key, c is the vaule
+#   creates a hash { [a,b] => c }
+#
+#  4 column e.g. a,b,c,d
+#   a,b  becomes the key, c,d the value
+#   creates a hash { [a,b] => [c,d] }
+#
+#  TODO allow mapping file to be an xml file
+#
+class ValueMapFromFile < Hash
+
+  def intialize(file_path, delim = ',')
+    @delegate_to = {}
+    @delim = delim
+    load_map(file_path)
+  end
+
+  def load_map(file_path = nil, delim = ',')
+    @file = file_path unless file_path.nil?
+    @delim = delim
+
+    raise ArgumentError.new("Can not read map file: #{@file}") unless File.readable?(@file)
+
+    File.open(@file).each_line do |line|
+      next unless(line && line.chomp!)
+
+      values = line.split(@delim)
+
+      case values.nitems
+        when 2 then self.store(values[0], values[1])
+        when 3 then self.store([values[0], values[1]], values[2])
+        when 4 then self.store([values[0], values[1]],[values[2], values[3]])
+      else
+        raise ArgumentError.new("Bad key,value row in #{@file}: #{values.nitems} number of columns not supported")
+      end
+    end
+
+    return self
+  end
+end
+
+
+# Expects file of format [TradeType,LDN_TradeId,HUB_TradeId,LDN_AssetId,HUB_AssetId,LDN_StrutureId,HUB_StructureId,LDN_ProductType,HUB_ProductType]
+# Convets in to and araya containing rows [LDN_TradeId, LDN_AssetId, HUB_TradeId, HUB_AssetId]
+class AssetMapFromFile < Array
+
+  def intialize(file_path, delim = ',')
+    @delegate_to = {}
+    @delim = delim
+    load_map(file_path)
+  end
+
+  def load_map(file_path = nil, delim = ',')
+    @file = file_path unless file_path.nil?
+    @delim = delim
+
+    raise ArgumentError.new("Can not read asset map file: #{@file}") unless File.readable?(@file)
+
+    File.open(@file).each_line do |line|
+      next unless(line && line.chomp!)
+      # skip the header row
+      next if line.include?('TradeType')
+
+      values = line.split(@delim)
+
+      self.push(Array[values[1], values[3], values[2], values[4]])
+    end
+
+    return self
+  end
+
+  def write_map(file_path = nil, delim = ',')
+    mapfile = File.open( file_path, 'w')
+
+    self.each{|row| mapfile.write(row.join(delim)+"\n")}
+  end
+
+end

data/lib/datashift/method_detail.rb

@@ -0,0 +1,237 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2010
+# License::   MIT
+#
+# Details::   This class provides info and access to the individual population methods
+#             on an AR model. Populated by, and coupled with MethodMapper,
+#             which does the model interrogation work and stores sets of MethodDetails.
+#
+#             Enables 'loaders' to iterate over the MethodMapper results set,
+#             and assign values to AR object, without knowing anything about that receiving object.
+#
+require 'to_b'
+
+module DataShift
+
+  class MethodDetail
+
+    def self.type_enum
+      @type_enum ||= Set[:assignment, :belongs_to, :has_one, :has_many]
+      @type_enum
+    end
+
+    # When looking up an association, try each of these in turn till a match
+    #  i.e find_by_name .. find_by_title and so on, lastly try the raw id
+    @@insistent_find_by_list ||= [:name, :title, :id]
+
+    # Name is the raw, client supplied name
+    attr_reader :name, :col_type, :current_value
+
+    attr_reader :operator, :operator_type
+
+    # Store the raw (client supplied) name against the active record  klass(model), operator and types
+    # col_types can typically be derived from klass.columns - set of ActiveRecord::ConnectionAdapters::Column
+
+    def initialize(client_name, klass, operator, type, col_types = {} )
+      @klass, @name = klass, client_name
+
+      if( MethodDetail::type_enum.member?(type.to_sym) )
+        @operator_type = type
+      else
+        raise "Bad operator Type #{type} passed to Method Detail"
+      end
+
+      @operator = operator
+    
+      # Note : Not all assignments will currently have a column type, for example
+      # those that are derived from a delegate_belongs_to
+      if(col_types.empty?)
+        @col_type = klass.columns.find{ |col| col.name == operator }
+      else
+        @col_type = col_types[operator]
+      end
+    end
+
+
+    # Return the actual operator's name for supplied method type
+    # where type one of :assignment, :has_one, :belongs_to, :has_many etc
+    def operator_for( type )
+      return operator if(@operator_type == type)
+      nil
+    end
+
+    def operator?(name)
+      operator == name
+    end
+
+    # Return the operator's expected class name, if can be derived, else nil
+    def operator_class_name()
+      @operator_class_name ||= if(operator_for(:has_many) || operator_for(:belongs_to) || operator_for(:has_one))
+        begin
+          Kernel.const_get(operator.classify)
+          operator.classify
+        rescue; ""; end
+  
+      elsif(@col_type)
+        @col_type.type.to_s.classify
+      else
+        ""
+      end
+
+      @operator_class_name
+    end
+
+    # Return the operator's expected class, if can be derived, else nil
+    def operator_class()
+      @operator_class ||= if(operator_for(:has_many) || operator_for(:belongs_to) || operator_for(:has_one))
+        begin
+          Kernel.const_get(operator.classify)
+        rescue; ""; end
+
+      elsif(@col_type)
+        begin
+          Kernel.const_get(@col_type.type.to_s.classify)
+        rescue; nil; end
+      else
+        nil
+      end
+
+      @operator_class
+    end
+
+
+    def assign(record, value )
+      
+      @current_value = value
+
+      puts "WARNING nil value supplied for Column [#{@name}]" if(@current_value.nil?)
+    
+      if( operator_for(:belongs_to) )
+      
+        #puts "DEBUG : BELONGS_TO : #{@name} : #{operator} - Lookup #{@current_value} in DB"
+        insistent_belongs_to(record, @current_value)
+
+      elsif( operator_for(:has_many) )
+
+        #puts "DEBUG : HAS_MANY :  #{@name} : #{operator}(#{operator_class}) - Lookup #{@current_value} in DB"
+        if(value.is_a?(Array) || value.is_a?(operator_class))
+          record.send(operator) << value
+        else
+          puts "ERROR #{value.class} - Not expected type for has_many #{operator} - cannot assign"
+          # TODO -  Not expected type - maybe try to look it up somehow ?"
+          #insistent_has_many(record, @current_value)
+        end
+
+      elsif( operator_for(:has_one) )
+
+        #puts "DEBUG : HAS_MANY :  #{@name} : #{operator}(#{operator_class}) - Lookup #{@current_value} in DB"
+        if(value.is_a?(operator_class))
+          record.send(operator + '=', value)
+        else
+          puts "ERROR #{value.class} - Not expected type for has_one #{operator} - cannot assign"
+          # TODO -  Not expected type - maybe try to look it up somehow ?"
+          #insistent_has_many(record, @current_value)
+        end
+
+      elsif( operator_for(:assignment) && @col_type )
+        #puts "DEBUG : COl TYPE defined for #{@name} : #{@assignment} => #{@current_value} #{@col_type.type}"
+        #puts "DEBUG : COl TYPE CAST: #{@current_value} => #{@col_type.type_cast( @current_value ).inspect}"
+        record.send( operator + '=' , @col_type.type_cast( @current_value ) )
+
+        #puts "DEBUG : MethodDetails Assignment RESULT: #{record.send(operator)}"
+
+      elsif( operator_for(:assignment) )
+        #puts "DEBUG : Brute force assignment of value  #{@current_value} supplied for Column [#{@name}]"
+        # brute force case for assignments without a column type (which enables us to do correct type_cast)
+        # so in this case, attempt straightforward assignment then if that fails, basic ops such as to_s, to_i, to_f etc
+        insistent_assignment(record, @current_value)
+      else
+        puts "WARNING: No operator found for assignment on #{self.inspect} for Column [#{@name}]"
+      end
+    end
+
+    def pp
+      "#{@name} => #{operator}"
+    end
+
+
+    def self.insistent_method_list
+      @insistent_method_list ||= [:to_s, :to_i, :to_f, :to_b]
+      @insistent_method_list
+    end
+
+    private
+
+    # Attempt to find the associated object via id, name, title ....
+    def insistent_belongs_to( record, value )
+
+      if( value.class == operator_class)
+        record.send(operator) << value
+      else
+
+        @@insistent_find_by_list.each do |x|
+          begin
+            next unless operator_class.respond_to?( "find_by_#{x}" )
+            item = operator_class.send( "find_by_#{x}", value)
+            if(item)
+              record.send(operator + '=', item)
+              break
+            end
+          rescue => e
+            puts "ERROR: #{e.inspect}"
+            if(x == MethodDetail::insistent_method_list.last)
+              raise "I'm sorry I have failed to assign [#{value}] to #{@assignment}" unless value.nil?
+            end
+          end
+        end
+      end
+    end
+
+    # Attempt to find the associated object via id, name, title ....
+    def insistent_has_many( record, value )
+
+      if( value.class == operator_class)
+        record.send(operator) << value
+      else
+        @@insistent_find_by_list.each do |x|
+          begin
+            item = operator_class.send( "find_by_#{x}", value)
+            if(item)
+              record.send(operator) << item
+              break
+            end
+          rescue => e
+            puts "ERROR: #{e.inspect}"
+            if(x == MethodDetail::insistent_method_list.last)
+              raise "I'm sorry I have failed to assign [#{value}] to #{operator}" unless value.nil?
+            end
+          end
+        end
+      end
+    end
+
+    def insistent_assignment( record, value )
+      #puts "DEBUG: RECORD CLASS #{record.class}"
+      op = operator + '='
+    
+      begin
+        record.send(op, value)
+      rescue => e
+        MethodDetail::insistent_method_list.each do |f|
+          begin
+            record.send(op, value.send( f) )
+            break
+          rescue => e
+            #puts "DEBUG: insistent_assignment: #{e.inspect}"
+            if f == MethodDetail::insistent_method_list.last
+              puts  "I'm sorry I have failed to assign [#{value}] to #{operator}"
+              raise "I'm sorry I have failed to assign [#{value}] to #{operator}" unless value.nil?
+            end
+          end
+        end
+      end
+    end
+  end
+  
+end

data/lib/datashift/method_mapper.rb

@@ -0,0 +1,257 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2010
+# License::   MIT
+#
+# Details::   A base class that stores details of all possible associations on AR classes and,
+#             given user supplied class and name, attempts to find correct attribute/association.
+#
+#             Derived classes define where the user supplied list of names originates from.
+#
+#             Example usage, load from a spreadsheet where the column names are only
+#             an approximation of the actual associations. Given a column heading of
+#             'Product Properties' on class Product,  find_method_detail() would search AR model,
+#             and return details of real has_many association 'product_properties'.
+#
+#             This real association can then be used to send spreadsheet row data to the AR object.
+#             
+require 'method_detail'
+
+module DataShift
+
+  class MethodMapper
+
+    attr_accessor :header_row, :headers
+    attr_accessor :method_details, :missing_methods
+  
+    @@has_many     = Hash.new
+    @@belongs_to   = Hash.new
+    @@assignments  = Hash.new
+    @@column_types = Hash.new
+
+    def initialize
+      @method_details = []
+      @headers = []
+    end
+
+    # Build complete picture of the methods whose names listed in method_list
+    # Handles method names as defined by a user or in file headers where names may
+    # not be exactly as required e.g handles capitalisation, white space, _ etc
+    # Returns: Array of matching method_details
+    #
+    def populate_methods( klass, method_list )
+      @method_details, @missing_methods = [], []
+    
+      method_list.each do |x|
+        md = MethodMapper::find_method_detail( klass, x )
+        md ? @method_details << md : @missing_methods << x
+      end
+      #@method_details.compact!  .. currently we may neeed to map via the index on @method_details so don't remove nils for now
+      @method_details
+    end
+
+    # The raw client supplied names
+    def method_names()
+      @method_details.collect( &:name )
+    end
+
+    # The true operator names discovered from model
+    def operator_names()
+      @method_details.collect( &:operator )
+    end
+
+    # Returns true if discovered methods contain every operator in mandatory_list
+    def contains_mandatory?( mandatory_list )
+      [ [*mandatory_list] - operator_names].flatten.empty?
+    end
+
+    def missing_mandatory( mandatory_list )
+      [ [*mandatory_list] - operator_names].flatten
+    end
+
+    # Create picture of the operators for assignment available on an AR model,
+    # including via associations (which provide both << and = )
+    # Options:
+    # :reload => clear caches and reperform  lookup
+    # :instance_methods => if true include instance method type assignment operators as well as model's pure columns
+    #
+    def self.find_operators(klass, options = {} )
+
+      # Find the has_many associations which can be populated via <<
+      if( options[:reload] || @@has_many[klass].nil? )
+        @@has_many[klass] = klass.reflect_on_all_associations(:has_many).map { |i| i.name.to_s }
+        klass.reflect_on_all_associations(:has_and_belongs_to_many).inject(@@has_many[klass]) { |x,i| x << i.name.to_s }
+      end
+      # puts "DEBUG: Has Many Associations:", @@has_many[klass].inspect
+
+      # Find the belongs_to associations which can be populated via  Model.belongs_to_name = OtherArModelObject
+      if( options[:reload] || @@belongs_to[klass].nil? )
+        @@belongs_to[klass] = klass.reflect_on_all_associations(:belongs_to).map { |i| i.name.to_s }
+      end
+
+      #puts "Belongs To Associations:", @@belongs_to[klass].inspect
+
+      # Find the has_one associations which can be populated via  Model.has_one_name = OtherArModelObject
+      if( options[:reload] || self.has_one[klass].nil? )
+        self.has_one[klass] = klass.reflect_on_all_associations(:has_one).map { |i| i.name.to_s }
+      end
+
+      #puts "has_one Associations:", self.has_one[klass].inspect
+
+      # Find the model's column associations which can be populated via xxxxxx= value
+      # Note, not all reflections return method names in same style so we convert all to
+      # the raw form i.e without the '='  for consistency 
+      if( options[:reload] || @@assignments[klass].nil? )
+
+        @@assignments[klass] = klass.column_names
+           
+        if(options[:instance_methods] == true)
+          setters = klass.instance_methods.grep(/\w+=/).collect {|x| x.to_s }
+
+          if(klass.respond_to? :defined_activerecord_methods)
+            setters = setters - klass.defined_activerecord_methods.to_a
+          end
+
+          # get into same format as other names 
+          @@assignments[klass] += setters.map{|i| i.gsub(/=/, '')}
+        end
+        
+        @@assignments[klass] -= @@has_many[klass] if(@@has_many[klass])
+        @@assignments[klass] -= @@belongs_to[klass] if(@@belongs_to[klass])
+        @@assignments[klass] -= self.has_one[klass] if(self.has_one[klass])
+ 
+        @@assignments[klass].uniq!
+
+        @@assignments[klass].each do |assign|
+          @@column_types[klass] ||= {}
+          column_def = klass.columns.find{ |col| col.name == assign }
+          @@column_types[klass].merge!( assign => column_def) if column_def
+        end
+      end
+    end
+
+    def self.build_method_details( klass )
+      @method_details ||= {}
+      
+      @method_details[klass] = []
+      
+      assignments_for(klass).each do |n|
+        @method_details[klass] << MethodDetail.new(n, klass, n, :assignment)
+      end
+        
+      has_one_for(klass).each do |n|
+        @method_details[klass] << MethodDetail.new(n, klass, n, :has_one)
+      end
+        
+      has_many_for(klass).each do |n|
+        @method_details[klass] << MethodDetail.new(n, klass, n, :has_many)
+      end
+        
+      belongs_to_for(klass).each do |n|
+        @method_details[klass] << MethodDetail.new(n, klass, n, :belongs_to)
+      end
+    end
+    
+    def self.method_details
+      @method_details ||= {}
+      @method_details
+    end
+    
+    # Find the proper format of name, appropriate call + column type for a given name.
+    # e.g Given users entry in spread sheet check for pluralization, missing underscores etc
+    #
+    # If not nil, returned method can be used directly in for example klass.new.send( call, .... )
+    #
+    def self.find_method_detail( klass, external_name )
+      operator = nil
+
+      name = external_name.to_s
+
+      # TODO - check out regexp to do this work better plus Inflections ??
+      # Want to be able to handle any of ["Count On hand", 'count_on_hand', "Count OnHand", "COUNT ONHand" etc]
+      [
+        name,
+        name.tableize,
+        name.gsub(' ', '_'),
+        name.gsub(' ', '_').downcase,
+        name.gsub(/(\s+)/, '_').downcase,
+        name.gsub(' ', ''),
+        name.gsub(' ', '').downcase,
+        name.gsub(' ', '_').underscore].each do |n|
+      
+        operator = (assignments_for(klass).include?(n)) ? n : nil
+      
+        return MethodDetail.new(name, klass, operator, :assignment, @@column_types[klass]) if(operator)
+
+        operator = (has_one_for(klass).include?(n)) ? n : nil
+      
+        return MethodDetail.new(name, klass, operator, :has_one, @@column_types[klass]) if(operator)
+
+        operator = (has_many_for(klass).include?(n)) ?  n : nil
+      
+        return MethodDetail.new(name, klass, operator, :has_many, @@column_types[klass]) if(operator)
+      
+        operator = (belongs_to_for(klass).include?(n)) ? n : nil
+      
+        return MethodDetail.new(name, klass, operator, :belongs_to, @@column_types[klass]) if(operator)
+      
+      end
+
+      nil
+    end
+
+    def self.clear
+      @@belongs_to.clear
+      @@has_many.clear
+      @@assignments.clear
+      @@column_types.clear
+      self.has_one.clear
+    end
+
+    def self.column_key(klass, column)
+      "#{klass.name}:#{column}"
+    end
+
+    # TODO - remove use of class variables - not good Ruby design
+    def self.belongs_to
+      @@belongs_to
+    end
+
+    def self.has_many
+      @@has_many
+    end
+
+    def self.has_one
+      @has_one ||= {}
+      @has_one
+    end
+
+    def self.assignments
+      @@assignments
+    end
+    def self.column_types
+      @@column_types
+    end
+
+
+    def self.belongs_to_for(klass)
+      @@belongs_to[klass] || []
+    end
+    def self.has_many_for(klass)
+      @@has_many[klass] || []
+    end
+
+    def self.has_one_for(klass)
+      self.has_one[klass] || []
+    end
+
+    def self.assignments_for(klass)
+      @@assignments[klass] || []
+    end
+    def self.column_type_for(klass, column)
+      @@column_types[klass] ?  @@column_types[klass][column] : []
+    end
+  
+  end
+
+end