datashift 0.15.0 → 0.16.0

data/lib/datashift/populator.rb

@@ -14,183 +14,287 @@ require 'logging'
 
 module DataShift
 
+  Struct.new("Substitution", :pattern, :replacement)
+
   class Populator
-    
+
     include DataShift::Logging
-        
+
     def self.insistent_method_list
       @insistent_method_list ||= [:to_s, :to_i, :to_f, :to_b]
     end
- 
+
     # When looking up an association, when no field provided, try each of these in turn till a match
     # i.e find_by_name, find_by_title, find_by_id
     def self.insistent_find_by_list
       @insistent_find_by_list ||= [:name, :title, :id]
     end
-    
-    
+
+
+    # Default data embedded in column headings - so effectively apply globally
+    # to teh whole column - hence class methods
+    def self.set_header_default_data(operator, data )
+      header_default_data[operator] = data
+    end
+
+    def self.header_default_data
+      @header_default_data ||= {}
+    end
+
+
     attr_reader :current_value, :original_value_before_override
+    attr_reader :current_col_type
+
     attr_reader :current_attribute_hash
     attr_reader :current_method_detail
-    
-    def initialize   
+
+    def initialize
       @current_value = nil
+      @current_method_detail = nil
       @original_value_before_override = nil
       @current_attribute_hash = {}
 
     end
-    
-    # Set member variables to hold details, value and optional attributes.
+
+    # Convert DSL string forms into a hash
+    # e.g
+    # 
+    #  "{:name => 'autechre'}" =>   Hash['name'] = autechre'
+    #  "{:cost_price => '13.45', :price => 23,  :sale_price => 4.23 }"
+
+    def self.string_to_hash( str )
+      h = {}
+      str.gsub(/[{}:]/,'').split(', ').map do |e|
+        k,v = e.split('=>')
+
+        k.strip!
+        v.strip!
+
+        if( v.match(/['"]/) )
+          h[k] = v.gsub(/["']/, '')
+        elsif( v.match(/^\d+$|^\d*\.\d+$|^\.\d+$/) )
+          h[k] = v.to_f
+        else
+          h[k] = v
+        end
+        h
+      end
+
+      h
+    end
+
+    # Set member variables to hold details, value and optional attributes,
+    # to be set on the 'value' once created
     # 
     # Check supplied value, validate it, and if required :
     #   set to provided default value
     #   prepend any provided prefixes 
     #   add any provided postfixes
     def prepare_data(method_detail, value)
-      
-      @current_value, @current_attribute_hash = value.to_s.split(Delimiters::attribute_list_start)
-      
-      if(@current_attribute_hash)
-        @current_attribute_hash.strip!
-        puts "DEBUG: Populator Value contains additional attributes"
-        @current_attribute_hash = nil unless @current_attribute_hash.include?('}')
-      end
-      
-      @current_attribute_hash ||= {}
-      
-      @current_method_detail = method_detail
-      
-      operator = method_detail.operator
-      
-      override_value(operator)
-        
-      if((value.nil? || value.to_s.empty?) && default_value(operator))
-        @current_value = default_value(operator)
+
+      raise NilDataSuppliedError.new("No method detail supplied for prepare_data") unless(method_detail)
+
+      begin
+        @prepare_data_const_regexp ||= Regexp.new( Delimiters::attribute_list_start + ".*" + Delimiters::attribute_list_end)
+
+        if( value.is_a? ActiveRecord::Relation ) # Rails 4 - query no longer returns an array
+          @current_value = value.to_a
+        elsif( !DataShift::Guards.jruby? && value.class.ancestors.include?(Spreadsheet::Formula))
+          @current_value = value.value
+        elsif( value.class.ancestors.include?(ActiveRecord::Base) || value.is_a?(Array))
+          @current_value = value
+        else
+          @current_value = value.to_s
+
+          attribute_hash = @current_value.slice!(@prepare_data_const_regexp)
+
+          if(attribute_hash)
+            @current_attribute_hash = Populator::string_to_hash( attribute_hash )
+            logger.info "Populator for #{@current_value} has attributes #{@current_attribute_hash.inspect}"
+          end
+        end
+
+        @current_attribute_hash ||= {}
+
+        @current_method_detail = method_detail
+
+        @current_col_type = @current_method_detail.col_type
+
+        operator = method_detail.operator
+
+        if(has_override?(operator))
+          override_value(operator)      # takes precedence over anything else
+        else
+          # if no value check for a defaults from config, headers
+          if(default_value(operator))
+            @current_value = default_value(operator)
+          elsif(Populator::header_default_data[operator])
+            @current_value = Populator::header_default_data[operator].to_s
+          elsif(Populator::header_default_data[operator])
+            @current_value = Populator::header_default_data[operator].to_s
+          elsif(method_detail.find_by_value)
+            @current_value = method_detail.find_by_value
+          end if(value.nil? || value.to_s.empty?)
+        end
+
+        substitute( operator )
+
+        @current_value = "#{prefix(operator)}#{@current_value}" if(prefix(operator))
+        @current_value = "#{@current_value}#{postfix(operator)}" if(postfix(operator))
+
+      rescue => e
+        logger.error("populator failed to prepare data supplied for operator #{method_detail.operator}")
+        logger.error("populator error: #{e.inspect}")
+        logger.error("populator stacktrace: #{e.backtrace.last}")
+        raise DataProcessingError.new("opulator failed to prepare data #{value} for operator #{method_detail.operator}")
       end
-      
-      @current_value = "#{prefix(operator)}#{@current_value}" if(prefix(operator))
-      @current_value = "#{@current_value}#{postfix(operator)}" if(postfix(operator))
 
       return @current_value, @current_attribute_hash
     end
-    
-    def assign(method_detail, record, value )
-     
-      @current_value = value
-       
-      # Rails 4 - not an array any more
-      if( value.is_a? ActiveRecord::Relation )
-        logger.warn("Relation passed rather than value #{value.inspect}")
-        @current_value = value.to_a
-      end
 
-      # logger.info("WARNING nil value supplied for Column [#{@name}]") if(@current_value.nil?)
+    # Main client hook
+
+    def prepare_and_assign(method_detail, record, value)
+
+      prepare_data(method_detail, value)
+
+      assign(record)
+    end
+
+    def assign(record)
+
+      raise NilDataSuppliedError.new("No method detail - cannot assign data") unless(current_method_detail)
+
+      operator = current_method_detail.operator
+
+      logger.debug("Populator assign - [#{current_value}] via #{current_method_detail.operator} (#{current_method_detail.operator_type})")
+
+      if( current_method_detail.operator_for(:belongs_to) )
+
+        insistent_belongs_to(current_method_detail, record, current_value)
+
+      elsif( current_method_detail.operator_for(:has_many) )
 
-      operator = method_detail.operator
-         
-      if( method_detail.operator_for(:belongs_to) )
-      
-        #puts "DEBUG : BELONGS_TO : #{@name} : #{operator} - Lookup #{@current_value} in DB"
-        insistent_belongs_to(method_detail, record, @current_value)
-
-      elsif( method_detail.operator_for(:has_many) )
-        
-        puts "DEBUG : VALUE TYPE [#{value.class.name.include?(operator.classify)}] [#{ModelMapper.class_from_string(value.class.name)}]" unless(value.is_a?(Array))
-     
         # The include? check is best I can come up with right now .. to handle module/namespaces
         # TODO - can we determine the real class type of an association
         # e.g given a association taxons, which operator.classify gives us Taxon, but actually it's Spree::Taxon
         # so how do we get from 'taxons' to Spree::Taxons ? .. check if further info in reflect_on_all_associations
 
-        if(@current_value.is_a?(Array) || @current_value.class.name.include?(operator.classify))
-          record.send(operator) << @current_value
-        else
-          puts "ERROR #{@current_value.class} - Not expected type for has_many #{operator} - cannot assign"
+        begin #if(current_value.is_a?(Array) || current_value.class.name.include?(operator.classify))
+          record.send(operator) << current_value
+        rescue => e
+          logger.error e.inspect
+          logger.error "Cannot assign #{current_value.inspect} (#{current_value.class}) to has_many association [#{operator}] "
         end
 
-      elsif( method_detail.operator_for(:has_one) )
+      elsif( current_method_detail.operator_for(:has_one) )
 
         #puts "DEBUG : HAS_MANY :  #{@name} : #{operator}(#{operator_class}) - Lookup #{@current_value} in DB"
-        if(@current_value.is_a?(method_detail.operator_class))
-          record.send(operator + '=', @current_value)
+        if(current_value.is_a?(current_method_detail.operator_class))
+          record.send(operator + '=', current_value)
         else
-          logger.error("ERROR #{value.class} - Not expected type for has_one #{operator} - cannot assign")
+          logger.error("ERROR #{current_value.class} - Not expected type for has_one #{operator} - cannot assign")
           # TODO -  Not expected type - maybe try to look it up somehow ?"
-          #insistent_has_many(record, @current_value)
         end
 
-      elsif( method_detail.operator_for(:assignment) && method_detail.col_type )
-        #puts "DEBUG : COl TYPE defined for #{@name} : #{@assignment} => #{@current_value} #{@col_type.type}"
-        # puts "DEBUG : Column [#{@name}] : COl TYPE CAST: #{@current_value} => #{@col_type.type_cast( @current_value ).inspect}"
-        record.send( operator + '=' , method_detail.col_type.type_cast( @current_value ) )
-
-        #puts "DEBUG : MethodDetails Assignment RESULT: #{record.send(operator)}"
+      elsif( current_method_detail.operator_for(:assignment) && current_col_type)
+        # 'type_cast' was changed to 'type_cast_from_database'
+        if Rails::VERSION::STRING < '4.2.0'
+          logger.debug("Assign #{current_value} => [#{operator}] (CAST 2 TYPE  #{current_col_type.type_cast( current_value ).inspect})")
+          record.send( operator + '=' , current_method_detail.col_type.type_cast( current_value ) )
+        else
+          logger.debug("Assign #{current_value} => [#{operator}] (CAST 2 TYPE  #{current_col_type.type_cast_from_database( current_value ).inspect})")
+          record.send( operator + '=' , current_method_detail.col_type.type_cast_from_database( current_value ) )
+        end
 
-      elsif( method_detail.operator_for(:assignment) )
-        #puts "DEBUG : Column [#{@name}] : Brute force assignment of value  #{@current_value}"
+      elsif( current_method_detail.operator_for(:assignment) )
+        logger.debug("Brute force assignment of value  #{current_value} => [#{operator}]")
         # brute force case for assignments without a column type (which enables us to do correct type_cast)
         # so in this case, attempt straightforward assignment then if that fails, basic ops such as to_s, to_i, to_f etc
-        insistent_assignment(record, @current_value, operator)
+        insistent_assignment(record, current_value, operator)
       else
         puts "WARNING: No assignment possible on #{record.inspect} using [#{operator}]"
         logger.error("WARNING: No assignment possible on #{record.inspect} using [#{operator}]")
       end
     end
-    
+
     def insistent_assignment(record, value, operator)
-      
-      #puts "DEBUG: RECORD CLASS #{record.class}"
+
       op = operator + '=' unless(operator.include?('='))
-    
+
       begin
         record.send(op, value)
       rescue => e
+
         Populator::insistent_method_list.each do |f|
           begin
             record.send(op, value.send( f) )
             break
           rescue => e
-            puts "DEBUG: insistent_assignment: #{e.inspect}"
             if f == Populator::insistent_method_list.last
-              puts  "I'm sorry I have failed to assign [#{value}] to #{operator}"
-              raise "I'm sorry I have failed to assign [#{value}] to #{operator}" unless value.nil?
+              logger.error(e.inspect)
+              logger.error("Failed to assign [#{value}] via operator #{operator}")
+              raise "Failed to assign [#{value}] to #{operator}" unless value.nil?
             end
           end
         end
       end
     end
-    
+
     # Attempt to find the associated object via id, name, title ....
     def insistent_belongs_to(method_detail, record, value )
 
       operator = method_detail.operator
-      
+
       if( value.class == method_detail.operator_class)
+        logger.info("Populator assigning #{value} to belongs_to association #{operator}")
         record.send(operator) << value
       else
 
-        insistent_find_by_list.each do |x|
-          begin
-            next unless method_detail.operator_class.respond_to?( "find_by_#{x}" )
-            item = method_detail.operator_class.send("find_by_#{x}", value)
-            if(item)
-              record.send(operator + '=', item)
-              break
-            end
-          rescue => e
-            logger.error("Attempt to find  associated object failed for #{method_detail}")
-            if(x == Populator::insistent_method_list.last)
-              raise "Populator failed to assign [#{value}] via moperator #{operator}" unless value.nil?
+        # TODO - DRY all this
+        if(method_detail.find_by_operator)
+
+          item = method_detail.operator_class.where(method_detail.find_by_operator => value).first_or_create
+
+          if(item)
+            logger.info("Populator assigning #{item.inspect} to belongs_to association #{operator}")
+            record.send(operator + '=', item)
+          else
+            logger.error("Could not find or create [#{value}] for belongs_to association [#{operator}]")
+            raise CouldNotAssignAssociation.new "Populator failed to assign [#{value}] to belongs_to association [#{operator}]"
+          end
+
+        else
+          #try the default field names
+          Populator::insistent_find_by_list.each do |x|
+            begin
+
+              next unless method_detail.operator_class.respond_to?("where")
+
+              item = method_detail.operator_class.where(x => value).first_or_create
+
+              if(item)
+                logger.info("Populator assigning #{item.inspect} to belongs_to association #{operator}")
+                record.send(operator + '=', item)
+                break
+              end
+            rescue => e
+              logger.error(e.inspect)
+              logger.error("Failed attempting to find belongs_to for #{method_detail.pp}")
+              if(x == Populator::insistent_method_list.last)
+                raise CouldNotAssignAssociation.new "Populator failed to assign [#{value}] to belongs_to association [#{operator}]" unless value.nil?
+              end
             end
           end
         end
+
       end
     end
-    
+
     def assignment( operator, record, value )
-      #puts "DEBUG: RECORD CLASS #{record.class}"
+
       op = operator + '=' unless(operator.include?('='))
-    
+
       begin
         record.send(op, value)
       rescue => e
@@ -199,7 +303,6 @@ module DataShift
             record.send(op, value.send( f) )
             break
           rescue => e
-            #puts "DEBUG: insistent_assignment: #{e.inspect}"
             if f == Populator::insistent_method_list.last
               puts  "I'm sorry I have failed to assign [#{value}] to #{operator}"
               raise "I'm sorry I have failed to assign [#{value}] to #{operator}" unless value.nil?
@@ -208,8 +311,8 @@ module DataShift
         end
       end
     end
-   
-    
+
+
     # Default values and over rides can be provided in Ruby/YAML ???? config file.
     # 
     #  Format :
@@ -224,80 +327,93 @@ module DataShift
     #
     def configure_from(load_object_class, yaml_file)
 
-      data = YAML::load( File.open(yaml_file) )
-      
-      # TODO - MOVE DEFAULTS TO OWN MODULE 
-      # decorate the loading class with the defaults/ove rides to manage itself
-      #   IDEAS .....
-      #
-      #unless(@default_data_objects[load_object_class])
-      #
-      #   @default_data_objects[load_object_class] = load_object_class.new
-      
-      #  default_data_object = @default_data_objects[load_object_class]
-      
-      
-      # default_data_object.instance_eval do
-      #  def datashift_defaults=(hash)
-      #   @datashift_defaults = hash
-      #  end
-      #  def datashift_defaults
-      #    @datashift_defaults
-      #  end
-      #end unless load_object_class.respond_to?(:datashift_defaults)
-      #end
-      
-      #puts load_object_class.new.to_yaml
-      
-      logger.info("Read Datashift loading config: #{data.inspect}")
-      
+      data = YAML::load( ERB.new( IO.read(yaml_file) ).result )
+
+      # TODO - MOVE DEFAULTS TO OWN MODULE
+
+      logger.info("Setting Populator defaults: #{data.inspect}")
+
       if(data[load_object_class.name])
-        
-        logger.info("Assigning defaults and over rides from config")
-        
+
         deflts = data[load_object_class.name]['datashift_defaults']
         default_values.merge!(deflts) if deflts
-        
+
+        logger.info("Set Populator default_values: #{default_values.inspect}")
+
         ovrides = data[load_object_class.name]['datashift_overrides']
         override_values.merge!(ovrides) if ovrides
+        logger.info("Set Populator overrides: #{override_values.inspect}")
+
+        subs = data[load_object_class.name]['datashift_substitutions']
+
+        subs.each do |o, sub|
+          # TODO support single array as well as multiple [[..,..], [....]]
+          sub.each { |tuple| set_substitution(o, tuple) }
+        end if(subs)
+
       end
-      
 
     end
-    
+
+    # Set a default value to be used to populate Model.operator
+    # Generally defaults will be used when no value supplied.
+    def set_substitution(operator, value )
+      substitutions[operator] ||= []
+
+      substitutions[operator] << Struct::Substitution.new(value[0], value[1])
+    end
+
+    def substitutions
+      @substitutions ||= {}
+    end
+
+    def substitute( operator )
+      subs = substitutions[operator] || {}
+
+      subs.each do |s|
+        @original_value_before_override = @current_value
+        @current_value = @original_value_before_override.gsub(s.pattern.to_s, s.replacement.to_s)
+      end
+    end
+
+
     # Set a value to be used to populate Model.operator
     # Generally over-rides will be used regardless of what value caller supplied.
     def set_override_value( operator, value )
       override_values[operator] = value
     end
-    
+
     def override_values
       @override_values ||= {}
     end
-    
+
     def override_value( operator )
       if(override_values[operator])
         @original_value_before_override = @current_value
-      
+
         @current_value = @override_values[operator]
       end
     end
-    
+
+
+    def has_override?( operator )
+      return override_values.has_key?(operator)
+    end
+
     # Set a default value to be used to populate Model.operator
     # Generally defaults will be used when no value supplied.
     def set_default_value(operator, value )
       default_values[operator] = value
     end
-    
+
     def default_values
       @default_values ||= {}
     end
-    
+
     # Return the default value for supplied operator
     def default_value(operator)
       default_values[operator]
     end
-    
 
     def set_prefix( operator, value )
       prefixes[operator] = value
@@ -310,7 +426,7 @@ module DataShift
     def prefixes
       @prefixes ||= {}
     end
-    
+
     def set_postfix(operator, value )
       postfixes[operator] = value
     end
@@ -318,12 +434,11 @@ module DataShift
     def postfix(operator)
       postfixes[operator]
     end
-    
+
     def postfixes
       @postfixes ||= {}
     end
-    
-    
+
   end
-  
+
 end