datashift 0.16.0 → 0.40.0

data/lib/datashift/context_factory.rb

@@ -0,0 +1,88 @@
+# Copyright:: (c) Autotelik Media Ltd 2015
+# Author ::   Tom Statter
+# License::   MIT
+#
+# Details::   Creates, configures and caches objects for use by current context
+#
+# SomeModule::MyAboutToBeLoadedClass:
+#     datashift_defaults:
+#       password: 'ns_spree_123'
+#       available_on: <%= Time.now.to_s(:db) %>
+#       shipping_category: 'Default'
+#       phone: '9999999999'
+#
+#     datashift_substitutions:
+#       order: ['#', 'NUMBER-']
+#       "Lineitems": ['10PAC', '5PAC']
+#
+#     datashift_populators:
+#       promo_code: PromoCodePopulator
+#
+module DataShift
+  class ContextFactory
+
+    class << self; attr_accessor :config end
+
+    # Clear out map of operators to populator Class
+    def self.clear_populators
+      populators.clear
+    end
+
+    # Return map of operators to populator Class
+    def self.populators
+      @populators ||= {}
+    end
+
+    # Options :
+    #    formatter
+    #    populator
+    #
+    def self.configure(load_object_class, yaml_file)
+
+      @config = YAML.load( ERB.new( IO.read(yaml_file) ).result )
+
+      @config[:datashift_populators].each do |_operator, type|
+        populator = ::Object.const_get(type).new
+
+        populator.configure_from(load_object_class, yaml_file)
+
+        populators[@config[:datashift_populators]]
+      end if @config[:datashift_populators]
+
+    end
+
+    # Move to CONTEXT
+    #     @populator = if(options[:populator].is_a?(String))
+    #                    ::Object.const_get(options[:populator]).new
+    #                  elsif(options[:populator].is_a?(Class))
+    #                    options[:populator].new
+    #                  else
+    #                    DataShift::Populator.new
+    #                  end
+
+    # Set a Populator to be used against an INBOUND operator
+
+    class << self
+      attr_writer :global_populator_class
+    end
+
+    def self.global_populator_class
+      @global_populator_class || DataShift::Populator
+    end
+
+    def self.set_populator(method_binding, klass)
+      operator = method_binding.is_a?(DataShift::MethodBinding) ? method_binding.operator : method_binding
+      populators[operator] = klass
+    end
+
+    def self.get_populator(method_binding)
+
+      if(populators.key?(method_binding.operator))
+        return populators[method_binding.operator].new
+      end unless method_binding.nil? || method_binding.invalid?
+
+      global_populator_class.new
+    end
+
+  end
+end

data/lib/datashift/core_ext/array.rb

@@ -0,0 +1,15 @@
+Array.class_eval do
+
+  ARRAY_FWDABLE_EXCLUDED_METHODS = [
+    :class, :singleton_class, :clone, :dup, :initialize_dup, :initialize_clone,
+    :freeze, :methods, :singleton_methods, :protected_methods, :private_methods, :public_methods,
+    :instance_variables, :instance_variable_get, :instance_variable_set, :instance_variable_defined?,
+    :instance_of?, :kind_of?, :is_a?, :tap, :send, :public_send, :respond_to?, :respond_to_missing?,
+    :extend, :display, :method, :public_method, :define_singleton_method, :object_id, :equal?,
+    :instance_eval, :instance_exec, :__send__, :__id__
+  ].freeze
+
+  def self.delegated_methods_for_fwdable
+    Array.instance_methods - ARRAY_FWDABLE_EXCLUDED_METHODS
+  end
+end

data/lib/datashift/core_ext/csv_ext.rb

@@ -0,0 +1,46 @@
+# Copyright:: Autotelik Media Ltd
+# Author ::   Tom Statter
+#
+require 'csv'
+require 'datashift/column_packer'
+require 'datashift/model_methods/model_method'
+
+class CSV
+
+  include DataShift::ColumnPacker
+
+  # Convert an AR instance to a set of CSV columns
+  # Additional non instance data can be included by supplying list of methods to call
+  # on the record
+  #
+  # Returns the data added
+  #
+  def ar_to_csv(record, remove_list = [], method_list = [])
+
+    serializable_hash = record.serializable_hash(except: remove_list)
+
+    csv_data = serializable_hash.values.collect { |c| escape_for_csv(c) }
+
+    [*method_list].each do |x|
+      csv_data << escape_for_csv(record.send(x)) if record.respond_to?(x)
+    end
+
+    csv_data
+  end
+
+  def ar_to_row(record, remove_list = [], method_list = [])
+    add_row( ar_to_csv(record, remove_list, method_list) )
+  end
+
+  def ar_association_to_csv(record, model_method, _options = {})
+    # pack association instances into single column
+    csv_data = if DataShift::ModelMethod.association_type?(model_method.operator_type)
+                 record_to_column( record.send(model_method.operator) )
+               else
+                 escape_for_csv( record.send(model_method.operator) )
+               end
+
+    csv_data
+  end
+
+end

data/lib/datashift/core_ext/string.rb

@@ -0,0 +1,49 @@
+String.class_eval do
+  # Convert DSL string forms into a hash
+  # e.g
+  #
+  #  "{:name => 'autechre'}" =>   Hash['name'] = autechre'
+  #  "{:cost_price => '13.45', :price => 23,  :sale_price => 4.23 }"
+  #  "{:cost_price => '13.45', :price => 23,  :sale_price => 4.23 }"
+
+  def to_hash_object
+
+    h = {}
+
+    gsub(/[{}]/, '').split(',').each do |e|
+      e.strip!
+
+      k, v = if e.include?('=>')
+               e.split('=>')
+             else
+               e.split(': ')
+             end
+
+      k = k.gsub(/[:']/, '').strip # easier to treat all keys as strings
+      v = v.to_s.strip
+
+      h[k] = if v =~ /['"]/
+               v.gsub(/["']/, '')
+             elsif v =~ /^\d+$|^\d*\.\d+$|^\.\d+$/
+               v.to_f
+             else
+               v
+             end
+      h
+    end
+
+    h
+  end
+
+  TRUE_REGEXP = /^(yes|true|on|t|1|\-1)$/i
+  FALSE_REGEXP = /^(no|false|off|f|0)$/i
+
+  def to_b
+    case self
+      when TRUE_REGEXP then true
+      when FALSE_REGEXP then false
+      else
+        to_i != 0
+    end
+  end
+end

data/lib/datashift/core_ext/to_b.rb

@@ -0,0 +1,11 @@
+
+Object.class_eval do
+  def to_b
+    case self
+      when true, false then self
+      when nil then false
+      else
+        to_i != 0
+    end
+  end
+end

data/lib/datashift/delimiters.rb

@@ -1,21 +1,17 @@
-# Copyright:: (c) Autotelik Media Ltd 2011
+# Copyright:: (c) Autotelik Media Ltd 2016
 # Author ::   Tom Statter
-# Date ::     Aug 2010
+# Date ::     Aug 2016
 # License::   MIT
 #
 #  Details::  Module providing standard location for delimiters used in both export/import
-#  
-#             For example we support multiple entries in a single column, so the string 
+#
+#             For example we support multiple entries in a single column, so the string
 #             needs to be formatted with recognisable delimiters seperating each of the multiple values.
 #
 module DataShift
 
   module Delimiters
 
-    # I made these class methods, feeling delims are 'global'
-    # I dunno now if thats good pattern or not
-
-
     # As well as just the column name, support embedding find operators for that column
     # in the heading .. i.e Column header => 'BlogPosts:user_id'
     # ... association has many BlogPosts selected via find_by_user_id
@@ -23,18 +19,19 @@ module DataShift
     # in the heading .. i.e Column header => 'BlogPosts:user_name:John Smith'
     # ... association has many BlogPosts selected via find_by_user_name("John Smith")
     #
-    def self.column_delim
+    def column_delim
       @column_delim ||= ':'
       @column_delim
     end
 
-    def self.set_column_delim(x)  @column_delim = x; end
-
+    def column_delim=(x)
+      @column_delim = x
+    end
 
     # Support multiple associations being added to a base object to be specified in a single column.
-    # 
+    #
     # Entry represents the association to find via supplied name, value to use in the lookup.
-    # 
+    #
     # Default syntax :
     #
     #   Name1:value1, value2|Name2:value1, value2, value3|Name3:value1, value2
@@ -51,38 +48,42 @@ module DataShift
     #         => Find 3 different associations, perform lookup via column called Size
     #         => Jumper.properties << [ small, medium, large ]
     #
-    def self.name_value_delim
+    def name_value_delim
       @name_value_delim ||= ':'
       @name_value_delim
     end
 
-    def self.set_name_value_delim(x)  @name_value_delim = x; end
-
+    def name_value_delim=(x)
+      @name_value_delim = x
+    end
 
     # The simple seperator for a list of values whether it be
     #     "Colour:red,green,blue".split(Delimiters::multi_value_delim) => [red,green,blue]
     #     {name => value, n2 => v2}.split(Delimiters::multi_value_delim) => [ [name => value], [n2 => v2] ]
 
-    def self.multi_value_delim
+    def multi_value_delim
       @multi_value_delim ||= ','
     end
-    
-    def self.set_multi_value_delim(x) @multi_value_delim = x; end
+
+    def multi_value_delim=(x)
+      @multi_value_delim = x
+    end
 
     # Objects can be created with multiple facets in single columns.
     # In this example a single Product can be configured with a consolidated mime and print types
-    # 
-    # mime_type:jpeg,PDF ; print_type:colour	 equivalent to 
-    # 
+    #
+    # mime_type:jpeg,PDF ; print_type:colour	 equivalent to
+    #
     #   => mime_type:jpeg;print_type:colour | mime_type:PDF; print_type:colour
-    
-    def self.multi_facet_delim
+
+    def multi_facet_delim
       @multi_facet_delim ||= ';'
     end
-    
-    def self.setmulti_facet_delim(x) @multi_facet_delim = x; end
-   
-    
+
+    def setmulti_facet_delim(x)
+      @multi_facet_delim = x
+    end
+
     # Multiple objects can be embedded in single columns.
     # In this example a single Category column contains 3 separate entries, New, SecondHand, Retro
     # object creation/update via hash (which hopefully we should be able to just forward to AR)
@@ -90,39 +91,40 @@ module DataShift
     #      | Category |
     #      'name =>New, :a => 1, :b => 2|name => SecondHand, :a => 6, :b => 34|Name:Old, :a => 12, :b => 67', 'Next Column'
     #
-    def self.multi_assoc_delim
+    def multi_assoc_delim
       @multi_assoc_delim ||= '|'
       @multi_assoc_delim
     end
 
-    def self.set_multi_assoc_delim(x) @multi_assoc_delim = x; end
-    
-    
+    def multi_assoc_delim=(x)
+      @multi_assoc_delim = x
+    end
+
     # Delimiters for {:abc => 2, :efg => 'some text}
-    
-    def self.attribute_list_start 
-      @attribute_list_start ||= '{';
+
+    def attribute_list_start
+      @attribute_list_start ||= '{'
     end
 
-    def self.attribute_list_start=(x) @attribute_list_start = x; end
-    
-    def self.attribute_list_end
+    attr_writer :attribute_list_start
+
+    def attribute_list_end
       @attribute_list_end ||= '}'
     end
-    
-    def self.attribute_list_end=(x) 
-      @attribute_list_end = x; 
-    end
-  
-    def self.csv_delim
-      @csv_delim ||= ','
-      @csv_delim
+
+    attr_writer :attribute_list_end
+
+    attr_writer :text_delim
+
+    attr_writer :key_value_sep
+
+    def csv_delimiter
+      @csv_delimiter ||= ','
     end
-    
-    def self.csv_delim=(x) set_csv_delim(x); end
-    def self.set_csv_delim(x) @csv_delim = x; end
 
-    def self.eol
+    attr_writer :csv_delimiter
+
+    def eol
       "\n"
     end
 
@@ -131,20 +133,12 @@ module DataShift
       @text_delim ||= "\'"
     end
 
-    def text_delim=(x)
-      @text_delim = x
-    end
-
     # seperator for identifying normal key value pairs
 
-    def self.key_value_sep
-      @key_value_sep ||= "=>"   #TODO check Ruby version and use appropriate has style ?
-    end
-
-    def self.key_value_sep=(x)
-      @key_value_sep = x
+    def key_value_sep
+      @key_value_sep ||= ' ' # for now assume everyone wants newer less verbose style
     end
 
   end
 
-end
+end

data/lib/datashift/doc_context.rb

@@ -0,0 +1,137 @@
+# Copyright:: (c) Autotelik Media Ltd 2015
+# Author ::   Tom Statter
+# Date ::     Jan 2015
+# License::   MIT
+#
+# Details::   The document we are dealing with
+#             Holds the headers, load object and access to reports for this Document
+#             Holds the current node_context (e.g column being processed) in relation to the Document
+#
+module DataShift
+
+  class DocContext
+
+    include DataShift::Logging
+
+    attr_reader :klass
+
+    attr_accessor :load_object
+
+    # The current Node Context - method_binding (includes inbound_column), row, data
+    attr_accessor :node_context
+
+    # The inbound document headers
+    attr_accessor :headers
+
+    attr_accessor :progress_monitor, :reporters
+
+    def initialize( klass )
+      reset_klass(klass)
+
+      @headers = DataShift::Headers.new(:na)
+
+      @progress_monitor = ProgressMonitor.new
+
+      @reporters = [DataShift::Reporters::BasicStdoutReporter.new(@progress_monitor)]
+    end
+
+    def reset_klass( klass )
+      @klass = klass
+      reset
+    end
+
+    # Reset the database object to be populated
+    #
+    def reset(object = nil)
+      @node_context = DataShift::EmptyContext.new
+
+      @load_object =  DataShift::LoadObject.new(object || new_load_object)
+    end
+
+    def new_load_object
+      @load_object = klass.new
+      @load_object
+    end
+
+    def create_node_context(method_binding, row_idx, data)
+      @node_context = DataShift::NodeContext.new(self, method_binding, row_idx, data)
+      @node_context
+    end
+
+    # Only save object if all columns ok, or allow errors in individual columns
+    def all_or_nothing?
+      true
+      # TODO: - read in from configration
+    end
+
+    # TOFIX - use delegation to doc_context.progress_monitor
+    def loaded_count
+      progress_monitor.loaded_count
+    end
+
+    def failed_count
+      progress_monitor.failed_count
+    end
+
+    def current_errors
+      load_object.errors.full_messages
+    end
+
+    def errors?
+      !load_object.errors.empty? || progress_monitor.current_status == :failure
+    end
+
+    # This method usually called during processing to avoid errors with associations like
+    #   <ActiveRecord::RecordNotSaved: You cannot call create unless the parent is saved>
+    # If the object is still invalid at this point probably indicates compulsory
+    # columns on model have not been processed before associations on that model.
+    #
+    # You can provide a custom sort function to the Collection of model methods (which are comparable) to fix this.
+    #
+    def save_if_new
+      return unless load_object.new_record?
+
+      return save if load_object.valid?
+
+      raise DataShift::SaveError, "Cannot Save Invalid #{load_object.class} Record : #{current_errors}"
+    end
+
+    # Save the object and then report the outcome to ProgressMonitor, as either success or failure
+    #
+    def save_and_monitor_progress
+      if(errors? && all_or_nothing?)
+        # Error already logged with doc_context.failure
+        logger.warn "SAVE skipped due to Errors for Row #{node_context.row_index} - #{node_context.method_binding.spp}"
+      else
+        if save
+          @progress_monitor.success(load_object)
+
+          logger.info("Successfully Processed [#{node_context.method_binding.spp}]")
+          logger.info("Successfully SAVED Object #{@progress_monitor.success_inbound_count} - [#{load_object.id}]")
+        else
+
+          failed = FailureData.new(load_object, node_context, current_errors)
+
+          @progress_monitor.failure(failed)
+
+          logger.info("Failed to Process [#{node_context.method_binding.spp}]")
+          logger.info("Failed to SAVE Object #{@progress_monitor.success_inbound_count} - [#{load_object.inspect}]")
+        end
+      end
+    end
+
+    def save
+      return false unless  load_object
+
+      logger.debug("SAVING #{load_object.class} : #{load_object.inspect}")
+      begin
+        load_object.save!
+      rescue => e
+        logger.error( "Save Error : #{e.inspect} on #{load_object.class}")
+        logger.error(e.backtrace)
+        false
+      end
+    end
+
+  end
+end