datashift 0.16.0 → 0.40.0

data/lib/datashift/mandatory.rb

@@ -0,0 +1,39 @@
+# Copyright:: (c) Autotelik Media Ltd 2015
+# Author ::   Tom Statter
+# Date ::     Aug 2015
+# License::   MIT
+#
+module DataShift
+
+  class Mandatory
+
+    include DataShift::Logging
+
+    attr_reader :mandatory_columns, :missing_columns
+
+    def initialize(columns)
+
+      @mandatory_columns = columns
+
+      logger.info("Mandatory columns set to #{@mandatory_columns.inspect}") unless @mandatory_columns.empty?
+
+      @comparable_mandatory_columns = @mandatory_columns.collect(&:downcase)
+      @missing_columns = []
+    end
+
+    def empty?
+      @comparable_mandatory_columns.empty?
+    end
+
+    # Sets mandatory_columns
+    # Returns true if bound methods contain every method listed in Mandatory
+    #
+    def contains_all?( binder )
+      return true if(empty?)
+      @missing_columns = @comparable_mandatory_columns - binder.operator_names.collect(&:downcase)
+      @missing_columns.empty?
+    end
+
+  end
+
+end

data/lib/datashift/mapping/data_flow_schema.rb

@@ -0,0 +1,198 @@
+# Copyright:: (c) Autotelik Media Ltd 2016
+# Author ::   Tom Statter
+# Date ::     Feb 2016
+# License::   MIT
+#
+# Details::   You can build a FLow for import or export within a YAML config file
+#
+#             It supports a simple DSL in below format.
+#
+#             SYNTAX :
+#               Indentation, usually 2 spaces, or a 2 space TAB, is very important
+#               <> are used to illustrate the elements that accept free text
+# Nodes :
+#
+#   Since the order is important, i.e should be preserved so your columns come out in defined order a sequence
+#   should be used
+#
+# Node :
+#
+# Each Node can have these elements
+#
+#   :heading:
+#       source:  The column header
+#       destination:  The column header
+#
+#   :operator:   How to get the data - for export would be the method call on the model
+#
+# EXAMPLE:
+#
+# data_flow_schema:
+#   Project:
+#     nodes:
+#       - project:
+#         heading:
+#           source: "title"
+#           destination: "Title"
+#         operator: title
+#
+#       - project_owner_budget:
+#         heading:
+#           destination: "Budget"
+#         operator: owner.budget
+#
+require 'erubis'
+
+module DataShift
+
+  class DataFlowSchema
+
+    include DataShift::Logging
+
+    attr_reader :nodes, :raw_data, :yaml_data
+
+    def initialize
+      @nodes = DataShift::NodeCollection.new
+    end
+
+    def sources
+      @nodes.collect(&:method_binding).collect(&:source)
+    end
+
+    # Build the node collection from a Class, that is for each operator in scope
+    # create a method binding and a node context, and add to collection.
+    #
+    def prepare_from_klass( klass, doc_context = nil )
+
+      context = doc_context || DocContext.new(klass)
+
+      @nodes = DataShift::NodeCollection.new
+
+      klass_to_model_methods( klass ).each_with_index do |mm, i|
+        binding = MethodBinding.new(mm.operator, i, mm)
+
+        @nodes << DataShift::NodeContext.new(context, binding, i, nil)
+      end
+
+      @nodes
+    end
+
+    # Helpers for dealing with Active Record models and collections
+    # Catalogs the supplied Klass and builds set of expected/valid Headers for Klass
+    #
+    def klass_to_model_methods(klass)
+
+      op_types_in_scope = DataShift::Configuration.call.op_types_in_scope
+
+      collection = ModelMethods::Manager.catalog_class(klass)
+
+      model_methods = []
+
+      if collection
+
+        collection.each { |mm| model_methods << mm if(op_types_in_scope.include? mm.operator_type) }
+
+        remove = DataShift::Transformation::Remove.new
+
+        remove.unwanted_model_methods model_methods
+      end
+
+      model_methods
+    end
+
+    def prepare_from_file(file_name, locale_key = 'data_flow_schema')
+      @raw_data = File.read(file_name)
+      yaml = YAML.load(raw_data)
+
+      prepare_from_yaml(yaml, locale_key)
+    end
+
+    def prepare_from_string(text, locale_key = 'data_flow_schema')
+      @raw_data = text
+      yaml = YAML.load(text)
+
+      prepare_from_yaml(yaml, locale_key)
+    end
+
+    def prepare_from_yaml(yaml, locale_key = 'data_flow_schema')
+
+      @yaml_data = yaml
+
+      @nodes = NodeCollection.new
+
+      raise RuntimeError.new("Bad YAML syntax  - No key #{locale_key} found in #{yaml}") unless yaml[locale_key]
+
+      locale_section = yaml[locale_key]
+
+      class_name = locale_section.keys.first
+
+      klass = MapperUtils.class_from_string_or_raise(class_name)
+
+      # The over all doc context
+      doc = DocContext.new(klass)
+      nodes.doc_context = doc
+
+      klass_section = locale_section[class_name]
+
+      DataShift::Transformation.factory { |f| f.configure_from_yaml(class_name, klass_section) }
+
+      yaml_nodes = klass_section['nodes']
+
+      logger.info("Read Data Schema Nodes: #{yaml_nodes.inspect}")
+
+      unless(yaml_nodes.is_a?(Array))
+        Rails.logger.error('Bad syntax in flow schema YAML - Nodes should be a sequence')
+        raise 'Bad syntax in flow schema YAML - Nodes should be a sequence'
+      end
+
+      model_method_mgr = ModelMethods::Manager.catalog_class(klass)
+
+      yaml_nodes.each_with_index do |keyed_node, i|
+
+        unless(keyed_node.keys.size == 1)
+          Rails.logger.error('Bad syntax in flow schema YAML - Section should be keyed hash')
+          raise 'Bad syntax in flow schema YAML - Section should be keyed hash'
+        end
+
+        # data_flow_schema:
+        #   Project:
+        #     nodes:
+        #       - project:
+        #           heading:
+        #             source: "title"
+        #             presentation: "Title"
+        #           operator: title
+        #           operator_type: has_many
+        #
+        logger.info("Node Data: #{keyed_node.inspect}")
+
+        # type one of ModelMethod.supported_types_enum
+        section = keyed_node.values.first
+
+        if(section['operator'])
+          # Find the domain model method details
+          model_method = model_method_mgr.search(section['operator'])
+
+          unless model_method
+            operator_type = section['operator_type'] || :method
+
+            model_method = model_method_mgr.insert(section['operator'], operator_type)
+          end
+        end
+
+        source = section.fetch('heading', {}).fetch('source', nil)
+
+        doc.headers.add( source )
+
+        method_binding = MethodBinding.new(source, i, model_method)
+
+        node = DataShift::NodeContext.new(doc, method_binding, i, nil)
+
+        nodes << node
+      end
+
+      nodes
+    end
+
+  end
+end

data/lib/datashift/{model_mapper.rb → mapping/mapper_utils.rb}

@@ -1,24 +1,33 @@
+# Copyright:: (c) Autotelik Media Ltd 2015
+# Author ::   Tom Statter
+# Date ::     Feb 2015
+# License::   MIT
+#
+# Details::   Helpers for dealing with Ccass <-> String, Table <-> String type mappings
+
 module DataShift
 
-  class ModelMapper
+  class MapperUtils
 
+    def self.ensure_class( klass )
+      klass.is_a?(String) ? MapperUtils.class_from_string_or_raise( klass ) : klass
+    end
 
     def self.class_from_string_or_raise( klass )
 
       ruby_klass = begin
-                     # support modules e.g "Spree::Property")
-        ModelMapper::class_from_string(klass)  #Kernel.const_get(model)
+        # support modules e.g "Spree::Property")
+        MapperUtils.class_from_string(klass) # Kernel.const_get(model)
       rescue NameError => e
-        puts e
+        logger.error( e.message )
         raise Thor::Error.new("ERROR: No such Class [#{klass}] found - check valid model supplied")
       end
 
-      raise NoSuchClassError.new("ERROR: No such Model [#{klass}] found - check valid model supplied") unless(ruby_klass)
+      raise NoSuchClassError.new("ERROR: No such Model [#{klass}] found - check valid model supplied") unless ruby_klass
 
       ruby_klass
     end
 
-
     # Helper to deal with string versions of modules/namespaced classes
     # Find and return the base class from a string.
     #
@@ -30,18 +39,29 @@ module DataShift
       end
     end
 
-
     # Similar to const_get_from_string except this version
     # returns nil if no such class found
     # Support modules e.g "Spree::Property"
     #
     def self.class_from_string( str )
+
+      MapperUtils.const_get_from_string(str.to_s) # Kernel.const_get(model)
+    rescue
+      return nil
+
+    end
+
+    def self.table_to_arclass(table, mod)
+
+      find_table = mod.nil? ? table.classify : "#{mod}::#{table.classify}"
+
       begin
-        ModelMapper::const_get_from_string(str.to_s)  #Kernel.const_get(model)
+        DataShift::MapperUtils.class_from_string(find_table)
+      rescue LoadError
       rescue
-        return nil
+        nil
       end
     end
 
   end
-end
+end

data/lib/datashift/model_methods/catalogue.rb

@@ -0,0 +1,183 @@
+# Copyright:: (c) Autotelik Media Ltd 2015
+# Author ::   Tom Statter
+# Date ::     Feb 2015
+# License::   MIT
+#
+# Details::   Catalogues all possible methods for populating data on a class
+#
+#             Provides high level find facilities to perform cataloguing on a class
+#             and retrieving operators per type (has_one, has_many, belongs_to, instance_method etc)
+#
+module DataShift
+
+  module ModelMethods
+
+    # HIGH LEVEL COLLECTION METHODS
+
+    class Catalogue
+
+      include DataShift::Logging
+      extend DataShift::Logging
+
+      def self.catalogued?(klass)
+        catalogued.include?(klass)
+      end
+
+      def self.size
+        catalogued.size
+      end
+
+      # Create simple picture of all the operator names for assignment available on a domain model,
+      # grouped by type of association (includes belongs_to and has_many which provides both << and = )
+      # Options:
+      #   :reload => clear caches and re-perform  lookup
+      #   :instance_methods => if true include instance method type 'setters' as well as model's pure columns
+      #
+      def self.populate(klass, options = {} )
+
+        raise "Cannot find operators supplied klass nil #{klass}" if klass.nil?
+
+        register(klass)
+
+        logger.debug("Catalogue - building operators information for #{klass}")
+
+        # Find the has_many associations which can be populated via <<
+        if options[:reload] || has_many[klass].nil?
+          has_many[klass] = klass.reflect_on_all_associations(:has_many).map { |i| i.name.to_s }
+
+          klass.reflect_on_all_associations(:has_and_belongs_to_many).inject(has_many[klass]) do |x, i|
+            x << i.name.to_s
+          end
+        end
+
+        # Find the belongs_to associations which can be populated via  Model.belongs_to_name = OtherArModelObject
+        if options[:reload] || belongs_to[klass].nil?
+          belongs_to[klass] = klass.reflect_on_all_associations(:belongs_to).map { |i| i.name.to_s }
+        end
+
+        # Find the has_one associations which can be populated via  Model.has_one_name = OtherArModelObject
+        if options[:reload] || has_one[klass].nil?
+          has_one[klass] = klass.reflect_on_all_associations(:has_one).map { |i| i.name.to_s }
+        end
+
+        # Find the model's column associations which can be populated via xxxxxx= value
+        # Note, not all reflections return method names in same style so we convert all to
+        # the raw form i.e without the '='  for consistency
+        if options[:reload] || assignments[klass].nil?
+          build_assignments( klass, options[:instance_methods] )
+        end
+      end
+
+      def self.clear
+        belongs_to.clear
+        has_many.clear
+        assignments.clear
+        column_types.clear
+        has_one.clear
+        catalogued.clear
+      end
+
+      # rubocop:disable Style/PredicateName
+
+      def self.belongs_to
+        @belongs_to ||= {}
+        @belongs_to
+      end
+
+      def self.has_many
+        @has_many ||= {}
+        @has_many
+      end
+
+      def self.has_one
+        @has_one ||= {}
+        @has_one
+      end
+
+      def self.assignments
+        @assignments ||= {}
+        @assignments
+      end
+
+      # N.B this return strings for consistency with other collections
+      # Removes methods that start with '_'
+
+      def self.setters( klass )
+
+        @keep_only_pure_setters ||= Regexp.new(/^[a-zA-Z]\w+=/)
+
+        setters = klass.instance_methods.grep(@keep_only_pure_setters).sort.collect( &:to_s )
+        setters.uniq # TOFIX is this really required ?
+      end
+
+      def self.column_types
+        @column_types ||= {}
+        @column_types
+      end
+
+      def self.belongs_to_for(klass)
+        belongs_to[klass] || []
+      end
+
+      def self.has_many_for(klass)
+        has_many[klass] || []
+      end
+
+      def self.has_one_for(klass)
+        has_one[klass] || []
+      end
+
+      def self.assignments_for(klass)
+        assignments[klass] || []
+      end
+
+      def self.column_type_for(klass, column)
+        column_types[klass] ? column_types[klass][column] : []
+      end
+
+      # rubocop:enable Style/PredicateName
+
+      class << self
+        private
+
+        def build_assignments(klass, include_instance_methods)
+          begin
+            assignments[klass] = klass.column_names
+          rescue => x
+            raise DataShiftException, "Failed to process column_names for class #{klass} - #{x.message}"
+          end
+
+          # get into consistent format with other assignments names i.e remove the = for now
+          assignments[klass] += setters(klass).map { |i| i.delete('=') } if include_instance_methods
+
+          # Now remove all the associations
+          assignments[klass] -= has_many[klass]   if has_many[klass]
+          assignments[klass] -= belongs_to[klass] if belongs_to[klass]
+          assignments[klass] -= has_one[klass]    if has_one[klass]
+
+          # TODO: remove assignments with id
+          # assignments => tax_id  but already in belongs_to => tax
+
+          assignments[klass].uniq!
+
+          assignments[klass].each do |assign|
+            column_types[klass] ||= {}
+            column_def = klass.columns.find { |col| col.name == assign }
+            column_types[klass].merge!( assign => column_def) if column_def
+          end
+        end
+
+        def catalogued
+          @catalogued ||= []
+        end
+
+        def register(klass)
+          catalogued << klass
+          catalogued.uniq!
+        end
+      end
+
+    end
+
+  end
+end