datashift 0.15.0 → 0.16.0

data/lib/datashift/data_transforms.rb

@@ -0,0 +1,83 @@
+# Copyright:: (c) Autotelik Media Ltd 2015
+# Author ::   Tom Statter
+# Date ::     March 2015
+# License::   MIT
+#
+# Details::   Stores defaults, substitutions, over rides etc
+#             that can be applied to incoming data while being Populated
+#
+# WORK In PROGRESS
+
+module DataShift
+
+  module Transformations
+
+    # Default values and over rides can be provided in Ruby/YAML ???? config file.
+    #
+    #  Format :
+    #
+    #    Load Class:    (e.g Spree:Product)
+    #     datashift_defaults:
+    #       value_as_string: "Default Project Value"
+    #       category: reference:category_002
+    #
+    #     datashift_overrides:
+    #       value_as_double: 99.23546
+    #
+    #
+    #     datashift_substitutions:
+    #
+    #
+
+    class Base
+
+      include DataShift::Logging
+
+      # Map a Column to all relevant transforms
+
+      def configure_from(load_object_class, yaml_file)
+
+        data = YAML::load( ERB.new( IO.read(yaml_file) ).result )
+
+        if(data[load_object_class.name])
+        end
+      end
+
+      # Set a value to be used to populate Model.operator
+      # Generally over-rides will be used regardless of what value caller supplied.
+      def set( operator, value )
+        override_values[operator] = value
+      end
+
+      def transforms
+        @transforms ||= {}
+      end
+
+      def apply( operator, current_value )
+        if(transforms[operator])
+          perform_transformcurrent_value()
+        end
+      end
+
+      def has_transform?( operator )
+        return override_values.has_key?(operator)
+      end
+
+    end
+
+    class Substitution < Base
+
+      def type
+        :substitution
+      end
+    end
+
+    class Override < Base
+
+      def type
+        :override
+      end
+    end
+
+  end
+end

data/lib/datashift/delimiters.rb

@@ -10,9 +10,26 @@
 #
 module DataShift
 
- 
   module Delimiters
 
+    # I made these class methods, feeling delims are 'global'
+    # I dunno now if thats good pattern or not
+
+
+    # As well as just the column name, support embedding find operators for that column
+    # in the heading .. i.e Column header => 'BlogPosts:user_id'
+    # ... association has many BlogPosts selected via find_by_user_id
+    #
+    # in the heading .. i.e Column header => 'BlogPosts:user_name:John Smith'
+    # ... association has many BlogPosts selected via find_by_user_name("John Smith")
+    #
+    def self.column_delim
+      @column_delim ||= ':'
+      @column_delim
+    end
+
+    def self.set_column_delim(x)  @column_delim = x; end
+
 
     # Support multiple associations being added to a base object to be specified in a single column.
     # 
@@ -40,19 +57,31 @@ module DataShift
     end
 
     def self.set_name_value_delim(x)  @name_value_delim = x; end
-    # TODO - support embedded object creation/update via hash (which hopefully we should be able to just forward to AR)
-    #
-    #      |Category|
-    #      name:new{ :date => '20110102', :owner = > 'blah'}
-    #
-    
-    
+
+
+    # The simple seperator for a list of values whether it be
+    #     "Colour:red,green,blue".split(Delimiters::multi_value_delim) => [red,green,blue]
+    #     {name => value, n2 => v2}.split(Delimiters::multi_value_delim) => [ [name => value], [n2 => v2] ]
+
     def self.multi_value_delim
       @multi_value_delim ||= ','
-      @multi_value_delim
     end
     
     def self.set_multi_value_delim(x) @multi_value_delim = x; end
+
+    # Objects can be created with multiple facets in single columns.
+    # In this example a single Product can be configured with a consolidated mime and print types
+    # 
+    # mime_type:jpeg,PDF ; print_type:colour	 equivalent to 
+    # 
+    #   => mime_type:jpeg;print_type:colour | mime_type:PDF; print_type:colour
+    
+    def self.multi_facet_delim
+      @multi_facet_delim ||= ';'
+    end
+    
+    def self.setmulti_facet_delim(x) @multi_facet_delim = x; end
+   
     
     # Multiple objects can be embedded in single columns.
     # In this example a single Category column contains 3 separate entries, New, SecondHand, Retro
@@ -92,11 +121,30 @@ module DataShift
     
     def self.csv_delim=(x) set_csv_delim(x); end
     def self.set_csv_delim(x) @csv_delim = x; end
-    
+
     def self.eol
       "\n"
     end
 
+    # surround text in suitable quotes e.g "hello world, how are you" => ' "hello world, how are you" '
+    def text_delim
+      @text_delim ||= "\'"
+    end
+
+    def text_delim=(x)
+      @text_delim = x
+    end
+
+    # seperator for identifying normal key value pairs
+
+    def self.key_value_sep
+      @key_value_sep ||= "=>"   #TODO check Ruby version and use appropriate has style ?
+    end
+
+    def self.key_value_sep=(x)
+      @key_value_sep = x
+    end
+
   end
 
 end

data/lib/datashift/excel_base.rb

@@ -0,0 +1,123 @@
+# To change this template, choose Tools | Templates
+# and open the template in the editor.
+
+module ExcelBase
+
+
+
+  def sanitize_sheet_name( name )
+    name.gsub(/[\[\]:\*\/\\\?]/, '')
+  end
+
+
+  # TODO -revisit/refactor - maybe this should just be on the base class Excel
+  # so you can call direct like excel.parse_headers(options[:header_row])
+  # rather than
+  #   sheet = excel.worksheet( sheet_number )
+  #   parse_headers(sheet, options[:header_row])
+
+  attr_accessor :header_row_index, :excel_headers
+
+  def parse_headers( sheet,  header_row = 0 )
+
+    @header_row_index = header_row || 0
+
+    header_row = sheet.row(header_row_index)
+
+    raise MissingHeadersError, "No headers found - Check Sheet #{sheet} is complete and Row #{header_row_index} contains headers" unless(header_row)
+
+    @excel_headers = []
+
+    # TODO - make more robust - currently end on first empty column
+    # There is no actual max columns in Excel .. you will run out of memory though at some point
+    (0..1024).each do |column|
+      cell = header_row[column]
+      break unless cell
+      header = "#{cell.to_s}".strip
+      break if header.empty?
+      @excel_headers << header
+    end
+
+    @excel_headers
+  end
+
+  # Helpers for dealing with Active Record models and collections
+  # Specify array of operators/associations to include - possible values are :
+  #     [:assignment, :belongs_to, :has_one, :has_many]
+
+  def ar_to_headers( records, associations = nil, options = {} )
+    return if( !records.first.is_a?(ActiveRecord::Base) || records.empty?)
+
+    only = *options[:only] ? [*options[:only]] : nil
+
+    headers =[]
+
+    if associations
+      details_mgr = DataShift::MethodDictionary.method_details_mgrs[records.first.class]
+
+      [*associations].each do |a|
+
+        details_mgr.get_list(a).each do  |md|
+
+          next if(only && !only.include?( md.name.to_sym ) )
+
+          puts md.name.to_sym.inspect
+
+          headers << "#{md.operator}"
+
+        end
+      end if(details_mgr)
+
+    else
+
+      headers = records.first.class.columns.collect( &:name )
+    end
+
+    set_headers( headers )
+  end
+
+
+  # Pass a set of AR records
+  def ar_to_xls(records, options = {})
+    return if( ! records.first.is_a?(ActiveRecord::Base) || records.empty?)
+
+    row_index =
+        if(options[:no_headers])
+          0
+        else
+          ar_to_headers( records )
+          1
+        end
+
+    records.each do |record|
+      ar_to_xls_row(row_index, 0, record)
+
+      row_index += 1
+    end
+  end
+
+
+  # Save data from an AR record to the current row, based on the record's columns [c1,c2,c3]
+  # Returns the number of the final column written to  
+  def ar_to_xls_row(row, start_column, record)
+    return unless( record.is_a?(ActiveRecord::Base))
+
+    column = start_column
+    record.class.columns.each do |connection_column|
+      ar_to_xls_cell(row, column, record, connection_column)
+      column += 1
+    end
+    column
+  end
+
+  def ar_to_xls_cell(row, column, record, connection_column)
+    begin
+      datum = record.send(connection_column.name)
+
+      self[row, column] = datum
+    rescue => e
+      puts "Failed to export #{datum} from #{connection_column.inspect} to column #{column}"
+      puts e, e.backtrace
+    end
+  end
+end

data/lib/datashift/exceptions.rb

@@ -1,22 +1,60 @@
+# Copyright:: (c) Autotelik Media Ltd 2014 
+# Author ::   Tom Statter
+# Date ::     June 2014
+# License::   Free, Open Source.
+#
+
 module DataShift
+  
+  class DataShiftException < StandardError
+
+    include DataShift::Logging
+        
+    def initialize( msg )
+      super
+      logger.error( msg)
+    end
 
+    def self.generate name
+      new_class = Class.new(DataShiftException) do
+        def initialize( msg )
+          super( msg )
+        end
+      end
+
+      DataShift.const_set(name, new_class)
+    end
+  end
+
+  class NilDataSuppliedError  < DataShiftException
+    def initialize( msg )
+      super( msg )
+    end
+  end
+  
   class BadRuby < StandardError; end
   
   class UnsupportedFileType < StandardError; end
   class BadFile < StandardError; end
   
   class MappingDefinitionError < StandardError; end
-  class DataProcessingError < StandardError; end
+
 
   class MissingHeadersError < StandardError; end
   class MissingMandatoryError < StandardError; end
 
-  class RecordNotFound < StandardError; end
-  
   class PathError < StandardError; end
   
   class BadUri < StandardError; end
-   
-  class CreateAttachmentFailed < StandardError; end
-  
-end
+
+end
+
+DataShift::DataShiftException.generate( "ConnectionError")
+DataShift::DataShiftException.generate( "CouldNotAssignAssociation")
+DataShift::DataShiftException.generate( "CreateAttachmentFailed")
+DataShift::DataShiftException.generate( "DataProcessingError")
+DataShift::DataShiftException.generate( "FileNotFound")
+DataShift::DataShiftException.generate( "NoSuchClassError")
+DataShift::DataShiftException.generate( "MissingConfigOptionError")
+DataShift::DataShiftException.generate( "RecordNotFound")
+DataShift::DataShiftException.generate( "SaveError")

data/lib/datashift/load_object.rb

@@ -0,0 +1,25 @@
+# Copyright:: (c) Autotelik Media Ltd 2015
+# Author ::   Tom Statter
+# Date ::     March 2015
+# License::   MIT
+#
+# Details::   Manage the current loader object
+#
+require 'to_b'
+require 'logging'
+
+module DataShift
+
+  class LoadObject
+    
+    include DataShift::Logging
+
+    attr_accessor :load_object
+
+    def initialize( current_object = nil)
+      @load_object = current_object
+    end
+    
+  end
+  
+end

data/lib/datashift/mapping_service.rb

@@ -0,0 +1,91 @@
+# Copyright:: (c) Autotelik Media Ltd 2015
+# Author ::   Tom Statter
+# Date ::     Aug 2015
+# License::   MIT
+#
+# Details::   A cache type class that stores details of a source=>destination mapping
+#
+require 'erubis'
+
+module DataShift
+
+  class MappingService
+
+    include DataShift::Logging
+
+    # N.B :all_config, :key_config are OpenStruct data structure
+    # that provides definition of config entries as attributes with their accompanying values.
+    # So if you had a top level config entries in the YAML called path & full_name, you can call
+    #   config.path
+    #   config.full_name etc
+    #
+    # For a more Hash like representation use config.yaml or config[:attribute]
+
+    attr_reader :mapped_class_name, :map_file_name
+
+    attr_reader :raw_data, :yaml_data, :mapping_entry
+
+    def initialize( klass )
+      @mapped_class_name = klass.name
+    end
+
+    def read( file, key = nil )
+
+      @map_file_name = file
+
+      unless(map_file_name && File.exists?(map_file_name))
+        logger.error "Cannot open mapping file - #{map_file_name} - file does not exist."
+        raise FileNotFound.new("Cannot open mapping file - #{map_file_name}")
+      end
+
+      begin
+        # Load application configuration
+        set_mapping( map_file_name )
+
+        set_key_config!( key ) if key
+      rescue => e
+        puts e.inspect
+        logger.error "Failed to parse config file #{map_file_name} - bad YAML ?"
+        raise e
+      end
+    end
+
+    # OpenStruct not a hash .. supports form ... config.path, config.full_name etc
+    def method_missing(method, *args, &block)
+      #logger :debug, "method_missing called with : #{method}"
+      @mapping_entry.send(method)
+    end
+
+    private
+
+    def set_mapping( file )
+
+      @raw_data = File.read(file)
+
+      erb = begin
+       Erubis::Eruby.new(raw_data).result
+      rescue => e
+        puts "Failed to parse erb template #{file} error: #{e.inspect}"
+
+        logger.error "Config template error: #{e.inspect}"
+
+        raise e
+      end
+
+      begin
+        @yaml_data = YAML.load(erb)
+
+        logger.info "Loaded YAML config from [#{file}]"
+
+      rescue => e
+        puts "YAML parse error: #{e.inspect}"
+        logger.error "YAML parse error: #{e.inspect}"
+        raise e
+      end
+
+      @mapping_entry = OpenStruct.new(yaml_data)
+    end
+
+end
+
+end