datashift 0.16.0 → 0.40.0

data/lib/datashift/column_packer.rb

@@ -6,11 +6,14 @@
 # Details::   Helper for creating consistent import/export format
 #             of model's attributes/associations
 #
+require 'datashift/delimiters'
+
 module DataShift
 
   module ColumnPacker
 
     include Delimiters
+    extend Delimiters
 
     # Return opposite of text delim - "hello, 'barry'" => '"hello, "barry""'
     def escape_text_delim
@@ -18,111 +21,57 @@ module DataShift
       "\'"
     end
 
-
     # Ensure a value is written to CSV correctly
-    # TODO - better ways ?? - see transcoding and String#encode
+    # TODO: - better ways ?? - see transcoding and String#encode
 
     def escape_for_csv(value)
-      text = value.to_s.gsub(text_delim, escape_text_delim()).gsub("\n", "\\n")
+      return nil if value.blank?
+      text = value.to_s.gsub(text_delim, escape_text_delim).gsub("\n", '\\n')
 
-      text = "#{text_delim}#{text}#{text_delim}" if(text.include?(Delimiters::csv_delim))
+      text = "#{text_delim}#{text}#{text_delim}" if(text.include?(csv_delimiter) && text.present?)
       text
     end
 
-
-    def to_headers( records, associations = nil, options = {} )
-      return if( !records.first.is_a?(ActiveRecord::Base) || records.empty?)
-
-      only = *options[:only] ? [*options[:only]] : nil
-
-      headers =[]
-
-      if associations
-        details_mgr = DataShift::MethodDictionary.method_details_mgrs[records.first.class]
-
-        [*associations].each do |a|
-
-          details_mgr.get_list(a).each do  |md|
-
-            next if(only && !only.include?( md.name.to_sym ) )
-
-            headers << "#{md.operator}"
-
-          end
-        end if(details_mgr)
-
-      else
-
-        headers = records.first.class.columns.collect( &:name )
-      end
-
-      headers
-    end
-
-
     # Convert an AR instance to a single column
     #    e.g User  :  ":name = > 'tom', :role => 'developer'"
     #
     # OPTIONS
-    #     with_only  Specify (as symbols) columns for association types to export
-    #     json:         Export association data in single column in JSON format
     #
-    def record_to_column(record, options = {})
-
-      return "" if(record.nil? || (record.respond_to?(:each) && record.empty?) )
-
-      with_only = *options[:with_only] ? [*options[:with_only]] : nil
+    #     json:         Export association data in single column in JSON format
 
-      return record.to_json if(options[:json] && !with_only) # packs associations into single column
+    def record_to_column(record, json = false)
 
-      if( record.respond_to?(:each) )
+      return '' if record.nil? || (record.respond_to?(:each) && record.empty?)
 
-        return "" if(record.empty?)
+      return record.to_json if json # packs associations into single column
 
-        data = []
+      data = []
 
-        record.each { |r| data << record_to_column(r, options); }
+      if record.respond_to?(:each)
+        return '' if record.empty?
 
-        if(options[:json])
-          return data.to_json
-        else
-          return "#{data.join(Delimiters::multi_assoc_delim)}"
-        end
+        record.each { |r| data << record_to_column(r, json) }
 
+        data.join(multi_assoc_delim).to_s
       else
-
-        data = options[:json] ? {} : []
-
         record.serializable_hash.each do |name, value|
-          next if(with_only && !with_only.include?( name.to_sym ) )
-
-          if(options[:json])
-            data[name] = value
-          else
-            data << "#{name.to_sym} #{Delimiters::key_value_sep} #{value.to_s.gsub(text_delim, escape_text_delim)}"
-          end
-        end
-
-        if(options[:json])#
-          return data.to_json
-        else
-          "#{Delimiters::attribute_list_start}#{data.join(Delimiters::multi_value_delim)}#{Delimiters::attribute_list_end}"
+          text = value.to_s.gsub(text_delim, escape_text_delim)
+          data << "#{name}:#{key_value_sep}#{text}"
         end
 
+        "#{attribute_list_start}#{data.join(multi_value_delim)}#{attribute_list_end}"
       end
 
     end
 
-
     # Convert an AR instance to a set of CSV columns
     def record_to_csv(record, options = {})
       csv_data = record.serializable_hash.values.collect { |value| escape_for_csv(value) }
 
-      [*options[:methods]].each { |x| csv_data << escape_for_csv(record.send(x)) if(record.respond_to?(x)) } if(options[:methods])
+      [*options[:methods]].each { |x| csv_data << escape_for_csv(record.send(x)) if record.respond_to?(x) } if options[:methods]
 
-      csv_data.join( Delimiters::csv_delim )
+      csv_data.join( csv_delimiter )
     end
 
-
   end
 end

data/lib/datashift/configuration.rb

@@ -0,0 +1,317 @@
+# Copyright:: (c) Autotelik Media Ltd 2015
+# Author ::   Tom Statter
+# Date ::     Feb 2015
+# License::   MIT
+#
+# Details::   Read mappings and provide cache type services for source=>destination mappings
+#
+require 'erubis'
+
+module DataShift
+
+  class Configuration
+
+    # List of association +TYPES+ to INCLUDE [:assignment, :enum, :belongs_to, :has_one, :has_many, :method]
+    # Defaults to [:assignment, :enum]
+    #
+    # @param [Array<#call>] List of association Types to include (:has_one etc)
+    # @return [Array<#call>]
+    #
+    attr_accessor :with
+
+    # When calling the export with associations methods the default
+    # is to include ALL all association TYPES as defined by
+    #   ModelMethod.supported_types_enum
+    #
+    # This can be used to reduce this down to only export specific types
+    #
+    # @param [Array<#call>] List of association Types to EXCLUDE (:has_one etc)
+    # @return [Array<#call>]
+    #
+    attr_accessor :exclude
+
+    # @param [Array<#call>] List of columns to remove from  files
+    # @return [Array<#call>]
+    #
+    attr_accessor :remove_columns
+
+    # List of headers/columns that are Mandatory i.e must be present in the inbound data
+    #
+    # @param [Array<#call>] List of headers/columns that are Mandatory
+    # @return [Array<#call>]
+    #
+    attr_accessor :mandatory
+
+    # @param [Boolean] Remove standard Rails cols like :id, created_at etc
+    # Default is false - i.e id, created_at etc are included by default
+    # @return [Boolean]
+    #
+    attr_accessor :remove_rails
+
+    # When performing import, default is to ignore any columns that cannot be mapped  (via headers)
+    # To raise an error set strict => true
+    # Defaults to `false`. Set to `true` to cause exceptions to be thrown
+    # The setting is ignored if routes are disabled.
+    # @param [Boolean] value
+    # @return [Boolean]
+    attr_accessor :strict
+
+    # When performing writes use update methods that write immediately to DB
+    # and use validations.
+    #
+    # Validations can ensure business logic but this can be far less efficient as writes to DB once per column
+    #
+    # To raise an error set strict => true
+    # Default  is to use more efficient but less strict attribute writing,
+    # no write to DB/No validations run
+    # @param [Boolean] value
+    # @return [Boolean]
+    attr_accessor :update_and_validate
+
+    # Controls the amount of information written to the log
+    # Defaults to `false`. Set to `true` to cause extensive progress messages to be logged
+    # @param [Boolean] value
+    # @return [Boolean]
+    attr_accessor :verbose
+
+    # Do everything except commit changes.
+    # For import save will not be called on the final object
+    # Defaults to `false`. Set to `true` to cause extensive progress messages to be logged
+    # @param [Boolean] value
+    # @return [Boolean]
+    attr_accessor :dummy_run
+
+    # Expand association data into multiple columns
+    #
+    # @param [Boolean]
+    # @return [Boolean]
+    #
+    attr_accessor :expand_associations
+
+    # When importing/exporting associations default is to include ALL associations of included TYPES
+    #
+    # Specify associations by name to remove
+    #
+    # @param [Array<#call>] List of association Names to EXCLUDE
+    # @return [Array<#call>]
+    #
+    attr_accessor :exclude_associations
+
+    #  List of external columns that do not map to any operator but should be included in processing.
+    #
+    #  Example use cases
+    #
+    #  Provides the opportunity for loaders to provide specific methods to handle columns
+    #  that do not map directly to a model's operators or associations
+    #
+    #  Enable handling delegated methods i.e no direct association but method is on a model through it's delegate
+    #
+    # @param [Array] value
+    # @return [Array]
+    attr_accessor :force_inclusion_of_columns
+
+    #  All external columns should be included in processing whether or not they automatically map to an operator
+    #
+    # @param [Boolean]
+    # @return [Boolean]
+    #
+    attr_accessor :include_all_columns
+
+    def self.rails_columns
+      @rails_standard_columns ||= [:id, :created_at, :created_on, :updated_at, :updated_on]
+    end
+
+    def initialize
+      @with = [:assignment, :enum]
+      @exclude = []
+      @remove_columns = []
+
+      @strict = false
+      @verbose = false
+      @dummy_run = false
+      @force_inclusion_of_columns = []
+      @exclude_associations = []
+
+      @expand_associations = false
+
+      # default to more efficient attribute writing - no write to DB/no validations run
+      @update_and_validate = false
+    end
+
+    # @return [DataShift::Configuration] DataShift's current configuration
+    def self.call
+      @configuration ||= Configuration.new
+    end
+
+    def self.reset
+      @configuration = Configuration.new
+    end
+
+    # Set DataShift's configuration
+    # @param config [DataShift::Configuration]
+    class << self
+      attr_writer :configuration
+    end
+
+    # Modify DataShift's current configuration
+    # @yieldparam [DataShift::Configuration] config current DataShift config
+    # ```
+    # DataShift::Configuration.call do |config|
+    #   config.verbose = false
+    # end
+    # ```
+    def self.configure
+      yield call
+    end
+
+    # Prepare the operators types in scope based on number of configuration attributes
+    # Default is assignment only
+    #
+    # Responds to Configuration params :
+    #
+    #   with: [:assignment, :enum, :belongs_to, :has_one, :has_many, :method]
+    #
+    #   with: :all -> all op types
+    #
+    #   exclude: - Remove any of [::assignment, :enum, :belongs_to, :has_one, :has_many, :method]
+    #
+    def op_types_in_scope
+
+      types_in_scope = if with_all?
+                         ModelMethod.supported_types_enum.dup
+                       else
+                         [*@with].dup
+                       end
+
+      types_in_scope -= [*@exclude]
+
+      types_in_scope
+    end
+
+    def op_type_in_scope?( model_method )
+      op_types_in_scope.include? model_method.operator_type
+    end
+
+    def with_all?
+      [*@with].include?(:all)
+    end
+
+    # Take options and create a list of symbols to remove from headers
+    #
+    # Rails columns like id, created_at etc are included by default
+    # Specify option :remove_rails to remove them from output
+    #
+    def prep_remove_list
+      remove_list = [*remove_columns].compact.collect { |x| x.to_s.downcase.to_sym }
+
+      remove_list += DataShift::Configuration.rails_columns if remove_rails
+
+      remove_list
+    end
+
+    # Modify DataShift's current Export configuration from an options hash
+    def self.from_hash( options )
+      DataShift::Configuration.configure do |config|
+        options.each do |key, value|
+          config.send("#{key}=", value) if(config.respond_to?(key))
+        end
+      end
+    end
+
+  end
+
+  class YamlConfiguration
+
+    attr_accessor :datashift_defaults, :datashift_populators
+
+    include DataShift::Logging
+
+    # N.B @mappings is an OpenStruct data structure
+    #
+    attr_reader :key
+    attr_accessor :yaml_data, :key_config
+
+    def initialize( key = nil)
+      @key = key
+
+      @config = OpenStruct.new
+    end
+
+    def read( file, key = nil )
+
+      unless File.exist?(file)
+        logger.error "Cannot open configuration file - #{file} - file does not exist."
+        raise FileNotFound.new("Cannot open mapping file - #{file}")
+      end
+
+      begin
+        # Load application configuration
+        set_mapping( file )
+
+        set_key_config!( key ) if key
+      rescue => e
+        logger.error e.inspect
+        logger.error "Failed to parse config file #{map_file_name} - bad YAML ?"
+        raise e
+      end
+    end
+
+    # OpenStruct not a hash .. supports form ... config.path, config.full_name etc
+    def method_missing(method, *_args, &_block)
+      @config.send(method)
+    end
+
+    # Config file can contain sets of entries, identified with a key.
+    # e.g
+    #     A:
+    #       log_level: 1
+    #     B:
+    #       log_level: 2
+    #
+    # Restrict searches/config entries to single set via key e.g 'A'
+    #
+    def set_key_config!( key )
+      raise MissingConfigOptionError.new("No config entry found for key [#{key}]") unless yaml_data && yaml_data[key].is_a?(Hash)
+      @key_config = OpenStruct.new( yaml_config[key] ) # Argument HAS to be a hash
+    end
+
+    # Merge another YAML section (identified by key) into @key_config
+    def merge_key_config!( key )
+      raise Beeline::MissingConfigOptionError.new("No config entry found for key [#{key}]") unless yaml_data && yaml_data[key].is_a?(Hash)
+
+      temp = yaml_data[key].merge(@key_config.instance_variable_get('@table') || {})
+
+      key_config = OpenStruct.new(temp)
+    end
+
+    def set( key, value)
+      key_config.instance_variable_get('@table')[key] = value
+    end
+
+    private
+
+    def set_mapping( file )
+
+      erb = begin
+        Erubis::Eruby.new( File.read(file )).result
+      rescue => e
+        logger.error "Failed to parse erb template #{file} error: #{e.inspect}"
+        raise e
+      end
+
+      begin
+        yaml_data = YAML.load(erb)
+
+        logger.info "Loaded YAML config from [#{file}]"
+
+      rescue => e
+        logger.error "YAML parse error: #{e.inspect}"
+        raise e
+      end
+
+      @config = OpenStruct.new(yaml_data)
+    end
+
+  end
+
+end