datashift 0.16.0 → 0.40.0

data/lib/datashift/progress_monitor.rb

@@ -0,0 +1,91 @@
+# Copyright:: (c) Autotelik Media Ltd 2016
+# Author ::   Tom Statter
+# Date ::     Dec 2016
+# License::   MIT
+#
+# Details::   Store and report stats on the current load
+#
+module DataShift
+
+  class ProgressMonitor
+
+    include DataShift::Logging
+
+    # actual data rows/objects inbound
+    attr_accessor :processed_object_count
+    alias processed_inbound_count processed_object_count
+
+    # DB objects created, updated etc
+    attr_accessor :loaded_objects, :failed_objects
+
+    # Actual number of data rows processed
+    # Bearing in mind things like updates, these can be greater than loaded/failed objects
+    attr_accessor :success_inbound_count, :failed_inbound_count
+
+    attr_reader :current_status
+
+    def initialize
+      reset
+    end
+
+    def reset
+      @processed_object_count = 0
+      @loaded_objects = []
+      @failed_objects = []
+
+      @success_inbound_count = 0
+      @failed_inbound_count = 0
+
+      @current_status = :success
+    end
+
+    def start_monitoring
+      @current_status = :success
+    end
+
+    def success(reportable_object)
+      add_loaded_object(reportable_object)
+    end
+
+    # Loading failed. Store a failed object and if requested roll back (destroy) the current load object
+    # For use case where object saved early but subsequent required columns fail to process
+    # so the load object is invalid
+
+    def failure(failure_data)
+
+      @current_status = :failure
+
+      logger.error 'Failure(s) reported :'
+      [*failure_data.errors].each { |e| logger.error "\t#{e}" }
+
+      add_failed_object(failure_data)
+
+      failure_data.destroy_failed_object if(DataShift::Loaders::Configuration.call.destroy_on_failure)
+    end
+
+    def add_loaded_object(object)
+      @success_inbound_count += 1
+      @processed_object_count += 1
+
+      @loaded_objects << object.id unless object.nil? || @loaded_objects.include?(object)
+    end
+
+    def add_failed_object(object)
+      @failed_inbound_count += 1
+      @processed_object_count += 1
+
+      @failed_objects << object unless object.nil? || @failed_objects.include?(object)
+    end
+
+    # The database objects created or rejected
+
+    def loaded_count
+      loaded_objects.size
+    end
+
+    def failed_count
+      failed_objects.size
+    end
+
+  end
+end

data/lib/datashift/querying.rb

@@ -7,81 +7,130 @@
 #             based on a method map and supplied value from a file - i.e a single column/row's string value.
 #             Note that although a single column, the string can be formatted to contain multiple values.
 #
-#             Tightly coupled with MethodMapper classes (in lib/engine) which contains full details of
+#             Tightly coupled with Binder classes (in lib/engine) which contains full details of
 #             a file's column and it's correlated AR associations.
 #
 module DataShift
 
-  require 'datashift/method_mapper'
+  require 'datashift/binder'
 
   module  Querying
- 
+
+    include DataShift::Logging
+    extend DataShift::Logging
+
+    include Delimiters
+    extend Delimiters
+
+    # Return arrays 'where' clause field(s), and the value(s) to search for
+    #
+    # If not contained within inbound_data, check the Inbound Column headings
+    #
+    # Where field embedded in row ,takes precedence over field in column heading
+    #
+    # Treat rest of the node as the value to use in the where clause e.g
+    #   price:0.99 =>  Product.where( price: 0.99)
+    #
+    # Column headings will be used, if the row only contains data e.g
+    #   0.99
+    #
+    def self.where_field_and_values(method_binding, inbound_data)
+
+      part1, part2 = inbound_data.split(name_value_delim)
+
+      heading_lookups = method_binding.inbound_column.lookup_list
+
+      if (part1.nil? || part1.empty?) && (part2.nil? || part2.empty?)
+
+        # Column completely empty - check for lookup supplied via the
+        # inbound column headers/config
+
+        # TODO: - how best to support multipel lookup fields?
+        # part1 = heading_lookups.collect {|lookup| lookup.field }
+        part1 = heading_lookups.find_by_operator
+        part2 = heading_lookups.collect(&:value)
+
+      elsif part2.nil? || part2.empty?
+
+        # Only **value(s)** in column, so use field from header/config field
+
+        logger.debug("No part2e [#{part1}][#{part1.class}] - [#{part2}][#{part2.class}]")
+
+        part2 = part1.split(multi_value_delim)
+        part1 = method_binding.inbound_column.find_by_operator
+
+      else
+        part2 = part2.split(multi_value_delim)
+
+      end
+
+      logger.debug("Where clause [#{part1}][#{part1.class}] - [#{part2}][#{part2.class}]")
+
+      [part1, [*part2]]
+    end
+
     # Options:
-    # 
+    #
     #   :case_sensitive   : Default is a case insensitive lookup.
-    #   :use_like         : Attempts a lookup using ike and x% rather than equality 
+    #   :use_like         : Attempts a lookup using ike and x% rather than equality
     #
     def search_for_record(klazz, field, search_term, options = {})
-    
+
       begin
-        
-        if(options[:case_sensitive]) 
-          return klazz.send("find_by_#{field}", search_term)
-        elsif(options[:use_like])
-          return klazz.where("#{field} like ?", "#{search_term}%").first
-        else
-          return klazz.where("lower(#{field}) = ?", search_term.downcase).first
-        end
-     
+        return klazz.send("find_by_#{field}", search_term) if options[:case_sensitive]
+
+        return klazz.where("#{field} like ?", "#{search_term}%").first if options[:use_like]
+
+        return klazz.where("lower(#{field}) = ?", search_term.downcase).first
       rescue => e
-        puts e.inspect
-        logger.error("Exception attempting to find a record for [#{search_term}] on #{klazz}.#{field}")
-        logger.error e.backtrace
+        logger.error("Querying - Failed to find a record for [#{search_term}] on #{klazz}.#{field}")
         logger.error e.inspect
+        logger.error e.backtrace.last
       end
-      
+
       nil
     end
-    
+
     # Find a record for model klazz, looking up on field containing search_terms
     # Responds to global Options :
-    # 
+    #
     #   :add_prefix     : Add a prefix to each search term
     #   :case_sensitive : Default is a case insensitive lookup.
-    #   :use_like       : Attempts a lookup using like and x% rather than equality 
+    #   :use_like       : Attempts a lookup using like and x% rather than equality
     #
     # Returns nil if no record found
     def get_record_by(klazz, field, search_term, split_on = ' ', options = {})
-    
+
+      split_on_prefix = options[:add_prefix]
+
+      find_search_term = split_on_prefix ? "#{split_on_prefix}#{search_term}" : search_term
+
+      logger.info("Scanning for record where #{klazz}.#{field} ~=  #{find_search_term}")
+
       begin
-         
-        split_on_prefix = options[:add_prefix]
-        
-        z = (split_on_prefix) ? "#{split_on_prefix}#{search_term}": search_term
-        
-        logger.info("Scanning for record where #{klazz}.#{field} ~=  #{z}")
-        
-        record = search_for_record(klazz, field, z)
-        
-        # try individual portions of search_term, front -> back i.e "A_B_C_D" => A, B, C etc
-        search_term.split(split_on).each do |str|
-          z = (split_on_prefix) ? "#{split_on_prefix}#{str}": str
-          record = search_for_record(klazz, field, z, options)
-          break if record
-        end unless(record)
-        
+
+        record = search_for_record(klazz, field, find_search_term)
+
+        unless record
+          logger.info("Nothing found - trying split file_name to terms on [#{split_on}]")
+
+          # try individual portions of search_term, front -> back i.e "A_B_C_D" => A, B, C etc
+          search_term.split(split_on).each do |str|
+            find_search_term = split_on_prefix ? "#{split_on_prefix}#{str}" : str
+            logger.info("Scanning by term for record where #{field} ~=  #{find_search_term}")
+            record = search_for_record(klazz, field, find_search_term, options)
+            break if record
+          end
+        end
+
         # this time try incrementally scanning i.e "A_B_C_D" => A, A_B, A_B_C etc
-        search_term.split(split_on).inject("") do |str, term|
-          z = (split_on_prefix) ? "#{split_on_prefix}#{str}#{split_on}#{term}": "#{str}#{split_on}#{term}"
+        search_term.split(split_on).inject('') do |str, term|
+          z = split_on_prefix ? "#{split_on_prefix}#{str}#{split_on}#{term}" : "#{str}#{split_on}#{term}"
           record = search_for_record(klazz, field, z, options)
           break if record
           term
-        end unless(record)
-        
-        if(record && record.respond_to?(field)) 
-          logger.info("Record found for #{klazz}.#{field} : #{record.send(field)}" )  
-        end
-        
+        end unless record
+
         return record
       rescue => e
         logger.error("Exception attempting to find a record for [#{search_term}] on #{klazz}.#{field}")
@@ -90,14 +139,23 @@ module DataShift
         return nil
       end
     end
-    
+
     def get_record_by!(klazz, field, search_terms, split_on = ' ', options = {} )
       x = get_record_by(klazz, field, search_terms, split_on, options)
-      
-      raise RecordNotFound, "No #{klazz} record found for [#{search_terms}] on #{field}" unless(x)
-      
+
+      raise RecordNotFound, "No #{klazz} record found for [#{search_terms}] on #{field}" unless x
+
       x
     end
+
+    def find_or_new( klass, condition_hash = {} )
+      records = klass.where(condition_hash).all
+
+      return records.first if records.any?
+
+      klass.new
+    end
+
   end
 
-end
+end

data/lib/datashift/templates/import_export_config.erb

@@ -0,0 +1,55 @@
+# YAML Template for Datashift Import/Export Configuration
+#
+<%= @key -%>:
+  <%= @klass -%>:
+    defaults:
+      <%= "#<name>: <value>" if(@defaults.empty?) %>
+  <% @defaults.each do |k, v| -%>
+      <%= "#{k}: #{v}" %>
+  <% end %>
+
+    overrides:
+  <% if(@overrides.empty?) %>
+      <%= "#<name>: <value>" if(@overrides.empty?) %>
+  <% end %>
+  <% @overrides.each do |k, v| -%>
+      <% k %>: <% v %>
+  <% end -%>
+
+    # Expects a tuple (list with 2 entries), the rule and the replacement
+    substitutions:
+  <% if(@substitutions.empty?) %>
+      <%= "#- rule"  %>
+      <%= "#- replacement" %>
+  <% end %>
+  <% @substitutions.each do |k, v| -%>
+      <% k %>:
+        - <% v.first %>
+        - <% v.last %>
+  <% end -%>
+
+    prefixes:
+      <%= "#<name>: <value>" if(@prefixs.empty?) %>
+  <% @prefixs.each do |k, v| -%>
+      <%= k %>: <%= v %>
+  <% end -%>
+
+    postfixes:
+  <%= "    #<name>: <value>" if(@postfixs.empty?) %>
+  <% @postfixs.each do |k, v| -%>
+      <%= k %>: <%= v %>
+  <% end -%>
+
+  <% unless(@headers.empty?) %>
+    # Mappings between inbound column names and internal names
+    # are only required when datashift cannot guess the mapping itself
+    # It will automatically map headings like :
+    #  'Product properties' or 'Product_Properties', 'product Properties' etc to product_properties
+    nodes:
+    <% @headers.each_with_index do |s, i| %>
+      - <%= s %>:
+          heading:
+            source: <%= s %>
+            presentation: <%= s %>
+    <% end %>
+  <% end %>

data/lib/datashift/transformation/factory.rb

@@ -0,0 +1,219 @@
+# Copyright:: (c) Autotelik Media Ltd 2015
+# Author ::   Tom Statter
+# License::   MIT
+#
+# Details::   Maps transformations to internal Class methods.
+#
+#             Stores :
+#               substitutions
+#               over rides
+#               prefixes
+#               postfixes
+#
+# These are keyed on the associated method binding operator, which is
+# essentially the method call/active record column on the class.
+#
+# Clients can decide exactly how these can be applied to incoming data.
+#
+# Usage::
+#
+#   Provides a singleton instance of Transformations::Factory
+#   so you can specify additional transforms in .rb config as follows :
+#
+# IN : my_transformations.rb
+#
+#     DataShift::Transformer.factory do |factory|
+#        factory.set_default_on(Project, 'value_as_string', 'default text' )
+#     end
+#
+#   This global factory is automatically utilised by the default Populator
+#   during data load.
+#
+#   If passed an optional locale, rules for other
+#   languages can be specified. If not specified, defaults to <tt>:en</tt>.
+#
+require 'thread_safe'
+require 'active_support/inflector'
+
+# Helper class
+Struct.new('Substitution', :pattern, :replacement)
+
+module DataShift
+
+  module Transformation
+
+    extend self
+
+    # Yields a singleton instance of Transformations::Factory
+    # so you can specify additional transforms in .rb config
+    # If passed an optional locale, rules for other
+    # languages can be specified. If not specified, defaults to <tt>:en</tt>.
+    #
+    # Only rules for English are provided.
+    #
+    def factory(locale = :en)
+      if block_given?
+        yield Factory.instance(locale)
+      else
+        Factory.instance(locale)
+      end
+    end
+
+    class Factory
+
+      TRANSFORMERS_HASH_INSTANCE_NAMES = [:default, :override, :substitution, :prefix, :postfix].freeze
+
+      include DataShift::Logging
+
+      @__instance__ = ThreadSafe::Cache.new
+
+      def self.instance(locale = :en)
+        @__instance__[locale] ||= new
+      end
+
+      def self.reset(locale = :en)
+        @__instance__[locale] = new
+      end
+
+      attr_reader :defaults, :overrides, :substitutions
+      attr_reader :prefixes, :postfixes
+
+      def initialize
+        clear
+      end
+
+      def clear
+        TRANSFORMERS_HASH_INSTANCE_NAMES.each do|h|
+          instance_variable_set("@#{h.to_s.pluralize}", ActiveSupport::HashWithIndifferentAccess.new({}))
+        end
+      end
+
+      # Default values and over rides per class can be provided in YAML config file.
+      #
+      # The locale_key for situations where the Class one level down, for example as per our own
+      # Template for Datashift Import/Export Configuration which has format :
+      #
+      # key:
+      #  klass:
+      #
+      def configure_from(load_object_class, yaml_file, locale_key = nil)
+
+        data = YAML.load( ERB.new( IO.read(yaml_file) ).result )
+
+        class_name = load_object_class.name
+
+        data = data[locale_key] if(locale_key)
+
+        configure_from_yaml(load_object_class, data[class_name]) if(data[class_name])
+      end
+
+      def configure_from_yaml(load_object_class, yaml)
+
+        setter_method_map = {
+          defaults: :set_default_on,
+          overrides: :set_override_on,
+          substitutions: :set_substitution_on_list,
+          prefixes: :set_prefix_on,
+          postfixes: :set_postfix_on
+        }
+
+        setter_method_map.each do |key, call|
+          settings = yaml[key.to_s]
+
+          settings.each do |operator, value|
+            logger.info("Configuring Transform [#{key}] for [#{operator.inspect}] to [#{value}]")
+            send( call, load_object_class, operator, value)
+          end if(settings && settings.is_a?(Hash))
+        end
+
+      end
+
+      def hash_key(key)
+        return key if(key.is_a? String)
+        return key.class_name if(key.is_a? MethodBinding)
+        key.name # Class name
+      end
+
+      TRANSFORMERS_HASH_INSTANCE_NAMES.each do |tname|
+
+        plural_tname = tname.to_s.pluralize
+
+        # Example  : defaults_for(key)
+        #
+        class_eval <<-end_eval
+          def #{plural_tname}_for(key)
+           #{plural_tname}[hash_key(key)] ||= new_hash_instance
+          end
+        end_eval
+
+        # Is there a transform for this MethodBinding ?
+        #
+        # Example :  default?( method_binding )
+        #
+        class_eval <<-end_eval
+          def #{tname}?( method_binding )
+            #{plural_tname}_for(method_binding).key?(method_binding.operator)
+          end
+        end_eval
+
+        # RETURN a transform for this MethodBinding
+        #
+        # Example : default( method_binding )
+        #
+        class_eval <<-end_eval
+          def #{tname}( method_binding )
+           get_#{tname}_on(method_binding, method_binding.operator)
+          end
+        end_eval
+
+        # RETURN a transform for this key/operator pair
+        #
+        # Example : get_default_on(key, operator)
+        #
+        class_eval <<-end_eval
+          def get_#{tname}_on(key, operator)
+            #{plural_tname}_for(key)[operator]
+          end
+        end_eval
+
+        next if(tname == :substitution)
+
+        # Example : set_default_on(key, operator, default_value )
+        #
+        class_eval <<-end_eval
+          def set_#{tname}_on(key, operator, default_value )
+            #{plural_tname}_for(key)[operator] = default_value
+          end
+        end_eval
+
+        # Example : set_default(method_binding, default_value )
+        #
+        define_method("set_#{tname}") do |method_binding, default_value|
+          send("set_#{tname}_on", method_binding, method_binding.operator, default_value)
+        end
+
+      end
+
+      def set_substitution( method_binding, rule, replacement )
+        set_substitution_on( method_binding, method_binding.operator, rule, replacement)
+      end
+
+      def set_substitution_on(key, operator, rule, replacement )
+        substitutions_for(key)[operator] = Struct::Substitution.new(rule, replacement)
+      end
+
+      private
+
+      def set_substitution_on_list(key, operator, list )
+        substitutions_for(key)[operator] = Struct::Substitution.new(list[0], list[1])
+      end
+
+      def new_hash_instance
+        ActiveSupport::HashWithIndifferentAccess.new {}
+      end
+
+    end
+
+  end ## class
+
+end