datashift 0.16.0 → 0.40.0

data/lib/datashift/data_transforms.rb

@@ -1,83 +0,0 @@
-# Copyright:: (c) Autotelik Media Ltd 2015
-# Author ::   Tom Statter
-# Date ::     March 2015
-# License::   MIT
-#
-# Details::   Stores defaults, substitutions, over rides etc
-#             that can be applied to incoming data while being Populated
-#
-# WORK In PROGRESS
-
-module DataShift
-
-  module Transformations
-
-    # Default values and over rides can be provided in Ruby/YAML ???? config file.
-    #
-    #  Format :
-    #
-    #    Load Class:    (e.g Spree:Product)
-    #     datashift_defaults:
-    #       value_as_string: "Default Project Value"
-    #       category: reference:category_002
-    #
-    #     datashift_overrides:
-    #       value_as_double: 99.23546
-    #
-    #
-    #     datashift_substitutions:
-    #
-    #
-
-    class Base
-
-      include DataShift::Logging
-
-      # Map a Column to all relevant transforms
-
-      def configure_from(load_object_class, yaml_file)
-
-        data = YAML::load( ERB.new( IO.read(yaml_file) ).result )
-
-        if(data[load_object_class.name])
-        end
-      end
-
-      # Set a value to be used to populate Model.operator
-      # Generally over-rides will be used regardless of what value caller supplied.
-      def set( operator, value )
-        override_values[operator] = value
-      end
-
-      def transforms
-        @transforms ||= {}
-      end
-
-      def apply( operator, current_value )
-        if(transforms[operator])
-          perform_transformcurrent_value()
-        end
-      end
-
-      def has_transform?( operator )
-        return override_values.has_key?(operator)
-      end
-
-    end
-
-    class Substitution < Base
-
-      def type
-        :substitution
-      end
-    end
-
-    class Override < Base
-
-      def type
-        :override
-      end
-    end
-
-  end
-end

data/lib/datashift/mapping_file_definitions.rb

@@ -1,88 +0,0 @@
-# This class provides a value map (hash) from a text mapping file
-#
-# The map file is  a text file of delimeted key -> values pairs
-#
-#  SUPPORTED FILE FORMATS:
-#
-#  2 column e.g. a,b
-#   creates a simple hash {a => b)
-#
-#  3 column e.g. a,b,c
-#   a,b becomes the key, c is the vaule
-#   creates a hash { [a,b] => c }
-#
-#  4 column e.g. a,b,c,d
-#   a,b  becomes the key, c,d the value
-#   creates a hash { [a,b] => [c,d] }
-#
-#  TODO allow mapping file to be an xml file
-#
-class ValueMapFromFile < Hash
-
-  def intialize(file_path, delim = ',')
-    @delegate_to = {}
-    @delim = delim
-    load_map(file_path)
-  end
-
-  def load_map(file_path = nil, delim = ',')
-    @file = file_path unless file_path.nil?
-    @delim = delim
-
-    raise ArgumentError.new("Can not read map file: #{@file}") unless File.readable?(@file)
-
-    File.open(@file).each_line do |line|
-      next unless(line && line.chomp!)
-
-      values = line.split(@delim)
-
-      case values.nitems
-        when 2 then self.store(values[0], values[1])
-        when 3 then self.store([values[0], values[1]], values[2])
-        when 4 then self.store([values[0], values[1]],[values[2], values[3]])
-      else
-        raise ArgumentError.new("Bad key,value row in #{@file}: #{values.nitems} number of columns not supported")
-      end
-    end
-
-    return self
-  end
-end
-
-
-# Expects file of format [TradeType,LDN_TradeId,HUB_TradeId,LDN_AssetId,HUB_AssetId,LDN_StrutureId,HUB_StructureId,LDN_ProductType,HUB_ProductType]
-# Convets in to and araya containing rows [LDN_TradeId, LDN_AssetId, HUB_TradeId, HUB_AssetId]
-class AssetMapFromFile < Array
-
-  def intialize(file_path, delim = ',')
-    @delegate_to = {}
-    @delim = delim
-    load_map(file_path)
-  end
-
-  def load_map(file_path = nil, delim = ',')
-    @file = file_path unless file_path.nil?
-    @delim = delim
-
-    raise ArgumentError.new("Can not read asset map file: #{@file}") unless File.readable?(@file)
-
-    File.open(@file).each_line do |line|
-      next unless(line && line.chomp!)
-      # skip the header row
-      next if line.include?('TradeType')
-
-      values = line.split(@delim)
-
-      self.push(Array[values[1], values[3], values[2], values[4]])
-    end
-
-    return self
-  end
-
-  def write_map(file_path = nil, delim = ',')
-    mapfile = File.open( file_path, 'w')
-
-    self.each{|row| mapfile.write(row.join(delim)+"\n")}
-  end
-
-end

data/lib/datashift/mapping_service.rb

@@ -1,91 +0,0 @@
-# Copyright:: (c) Autotelik Media Ltd 2015
-# Author ::   Tom Statter
-# Date ::     Aug 2015
-# License::   MIT
-#
-# Details::   A cache type class that stores details of a source=>destination mapping
-#
-require 'erubis'
-
-module DataShift
-
-  class MappingService
-
-    include DataShift::Logging
-
-    # N.B :all_config, :key_config are OpenStruct data structure
-    # that provides definition of config entries as attributes with their accompanying values.
-    # So if you had a top level config entries in the YAML called path & full_name, you can call
-    #   config.path
-    #   config.full_name etc
-    #
-    # For a more Hash like representation use config.yaml or config[:attribute]
-
-    attr_reader :mapped_class_name, :map_file_name
-
-    attr_reader :raw_data, :yaml_data, :mapping_entry
-
-    def initialize( klass )
-      @mapped_class_name = klass.name
-    end
-
-    def read( file, key = nil )
-
-      @map_file_name = file
-
-      unless(map_file_name && File.exists?(map_file_name))
-        logger.error "Cannot open mapping file - #{map_file_name} - file does not exist."
-        raise FileNotFound.new("Cannot open mapping file - #{map_file_name}")
-      end
-
-      begin
-        # Load application configuration
-        set_mapping( map_file_name )
-
-        set_key_config!( key ) if key
-      rescue => e
-        puts e.inspect
-        logger.error "Failed to parse config file #{map_file_name} - bad YAML ?"
-        raise e
-      end
-    end
-
-    # OpenStruct not a hash .. supports form ... config.path, config.full_name etc
-    def method_missing(method, *args, &block)
-      #logger :debug, "method_missing called with : #{method}"
-      @mapping_entry.send(method)
-    end
-
-    private
-
-    def set_mapping( file )
-
-      @raw_data = File.read(file)
-
-      erb = begin
-       Erubis::Eruby.new(raw_data).result
-      rescue => e
-        puts "Failed to parse erb template #{file} error: #{e.inspect}"
-
-        logger.error "Config template error: #{e.inspect}"
-
-        raise e
-      end
-
-      begin
-        @yaml_data = YAML.load(erb)
-
-        logger.info "Loaded YAML config from [#{file}]"
-
-      rescue => e
-        puts "YAML parse error: #{e.inspect}"
-        logger.error "YAML parse error: #{e.inspect}"
-        raise e
-      end
-
-      @mapping_entry = OpenStruct.new(yaml_data)
-    end
-
-end
-
-end

data/lib/datashift/method_detail.rb

@@ -1,165 +0,0 @@
-# Copyright:: (c) Autotelik Media Ltd 2011
-# Author ::   Tom Statter
-# Date ::     Aug 2010
-# License::   MIT
-#
-# Details::   This class provides info and access to the details of individual population methods
-#             on an AR model. Populated by, and coupled with MethodMapper,
-#             which does the model interrogation work and stores sets of MethodDetails.
-#
-#             Enables 'loaders' to iterate over the MethodMapper results set,
-#             and assign values to AR object, without knowing anything about that receiving object.
-#
-require 'set'
-
-module DataShift
-
-  class MethodDetail
-
-    include DataShift::Logging
-      
-    def self.supported_types_enum
-      @type_enum ||= Set[:assignment, :belongs_to, :has_one, :has_many]
-      @type_enum
-    end
-
-    def self.association_types_enum
-      @assoc_type_enum ||= Set[:belongs_to, :has_one, :has_many]
-      @assoc_type_enum
-    end
-
-
-    def self.is_association_type? ( type )
-      association_types_enum.member?( type )
-    end
-
-
-    # Klass is the class of the 'parent' object i.e with the associations,
-    # For example Product which may have operator orders
-    attr_accessor :klass
-    
-    # Name is the raw, client supplied name e.g Orders
-    attr_accessor :name
-    attr_accessor :column_index
-  
-    # The rel col type from the DB
-    attr_reader :col_type
-
-    # The :operator that can be called to assign  e.g orders or Products.new.orders << Order.new
-    # 
-    # The type of operator e.g :assignment, :belongs_to, :has_one, :has_many etc
-    attr_reader :operator, :operator_type
-
-    # TODO make it a list/primary keys
-    # Additional helpers for where clauses
-    attr_accessor :find_by_operator, :find_by_value
-        
-    # Store the raw (client supplied) name against the active record  klass(model).
-    # Operator is the associated method call on klass,
-    # i.e client supplies name 'Price' in a spreadsheet, 
-    # but true operator to call on klass is price
-    # 
-    # col_types can typically be derived from klass.columns - set of ActiveRecord::ConnectionAdapters::Column
-
-    def initialize(client_name, klass, operator, type, col_types = {}, find_by_operator = nil, find_by_value = nil )
-      @klass, @name = klass, client_name
-      @find_by_operator = find_by_operator
-      @find_by_value = find_by_value
-
-      if( MethodDetail::supported_types_enum.member?(type.to_sym) )
-        @operator_type = type.to_sym
-      else
-        raise "Bad operator Type #{type} passed to Method Detail"
-      end
-
-      @operator = operator
-    
-      # Note : Not all assignments will currently have a column type, for example
-      # those that are derived from a delegate_belongs_to
-      if(col_types.empty?)
-        @col_type = klass.columns.find{ |col| col.name == operator }
-      else
-        @col_type = col_types[operator]
-      end
-      
-      @column_index = -1
-    end
-
-
-    # Return the actual operator's name for supplied method type
-    # where type one of :assignment, :has_one, :belongs_to, :has_many etc
-    def operator_for( type )
-      return operator if(@operator_type == type.to_sym)
-      nil
-    end
-
-    def operator?(name, case_sensitive = false)
-      case_sensitive ? operator == name : operator.downcase == name.downcase
-    end
-
-    # Return the operator's expected class name, if can be derived, else nil
-    def operator_class_name()
-      @operator_class_name ||= 
-        if(operator_for(:has_many) || operator_for(:belongs_to) || operator_for(:has_one))
-   
-        get_operator_class.name
-  
-      elsif(@col_type)
-        @col_type.type.to_s.classify
-      else
-        ""
-      end
-
-      @operator_class_name
-    end
-
-    # Return the operator's expected class, if can be derived, else nil
-    def operator_class()
-      @operator_class ||= get_operator_class()
-      @operator_class
-    end
-
-    def pp
-      "#{@name} => #{operator}"
-    end
-
-    private
-
-    # Return the operator's expected class, if can be derived, else nil
-    # TODO rspec- can reflect_on_association ever actually fail & do we ever need to try ourselves (badly)
-    def get_operator_class()
-      
-      if(operator_for(:has_many) || operator_for(:belongs_to) || operator_for(:has_one))
-
-        result = klass.reflect_on_association(operator)
-
-        return result.klass if(result)
-
-        result = ModelMapper::class_from_string(operator.classify)
-        
-        if(result.nil?)
-          begin
-            
-            first = klass.to_s.split('::').first
-            logger.debug "Trying to find operator class with Parent Namespace #{first}"
-
-            result = ModelMapper::const_get_from_string("#{first}::#{operator.classify}")
-          rescue => e
-            logger.error("Failed to derive Class for #{operator} (#{@operator_type} - #{e.inspect}")
-          end
-        end
-        
-        result
-
-      elsif(@col_type)
-        begin
-          Kernel.const_get(@col_type.type.to_s.classify)
-        rescue; nil; end
-      else
-        nil
-      end
-    end
-    
-  end
-  
-end

data/lib/datashift/method_details_manager.rb

@@ -1,95 +0,0 @@
-# Copyright:: (c) Autotelik Media Ltd 2011
-# Author ::   Tom Statter
-# Date ::     Aug 2010
-# License::   MIT
-#
-# Details::   This class provides info and access to groups of accessor methods,
-#             grouped by AR model. 
-#
-#             Stores complete collection of MethodDetail instances per mapped class.
-#             Provides high level find facilites to find a MethodDetail and to list out
-#             operators per type (has_one, has_many, belongs_to, instance_method etc) 
-#             and all possible operators, 
-
-require 'to_b'
-
-module DataShift
-
-  # Stores MethodDetails for a class mapped by type
-  class MethodDetailsManager
-
-    attr_reader :method_details, :method_details_list
-    attr_reader :managed_class_name
-    
-    def initialize( klass )
-      @managed_class_name = klass.name
-      @method_details = {}
-      @method_details_list = {}
-    end
-    
-    def add(method_details)
-      #puts "DEBUG: MGR - Add {#method_details.operator_type}\n#{method_details.inspect}"
-      @method_details[method_details.operator_type.to_sym] ||= {}
-      
-      # Mapped by Type and MethodDetail name
-      @method_details[method_details.operator_type.to_sym][method_details.name] = method_details
-      
-      # Helper list of all available by type
-      @method_details_list[method_details.operator_type.to_sym] ||= []
-       
-      @method_details_list[method_details.operator_type.to_sym] << method_details
-      @method_details_list[method_details.operator_type.to_sym].uniq!
-    end
-
-    def <<(method_details)
-      add(method_details)      
-    end
-
-    def find(name, type)
-      method_details = get(type)
-     
-      method_details ? method_details[name] : nil
-    end
-    
-    # type is expected to be one of MethodDetail::supported_types_enum
-    # Returns all MethodDetail(s) for supplied type
-    def get( type )
-      @method_details[type.to_sym]
-    end
-    
-    def get_list( type )
-      @method_details_list[type.to_sym] || []
-    end
-    
-    alias_method(:get_list_of_method_details, :get_list) 
-
-    # Get list of the inbound or externally supplied names
-    def get_names(type)
-      get_list(type).collect { |md| md.name }
-    end
-
-    # Get list of Rails model operators   
-    def get_operators(type)
-      get_list(type).collect { |md| md.operator }
-    end
-    
-    alias_method(:get_list_of_operators, :get_operators) 
-     
-    def available_operators
-      method_details_list.values.flatten.collect(&:operator)
-    end
-
-    # A reverse map  showing all operators with their associated 'type'
-    def available_operators_with_type
-      h = {}
-      method_details_list.each { |t, mds| mds.each do |v| h[v.operator] = t end }
-    
-      # this is meant to be more efficient that Hash[h.sort]
-      sh = {}
-      h.keys.sort.each do |k| sh[k] = h[k] end
-      sh
-    end
-    
-  end
-  
-end