datashift 0.16.0 → 0.40.0

data/lib/applications/jruby/word.rb

@@ -22,55 +22,55 @@
 # WORD_OLE_CONST.const_get( 'LesCR' ) or WORD_OLE_CONST::LesCR
 #   => 1
 
-if(Guards::windows?)
+if Guards.windows?
 
-require 'win32ole'
+  require 'win32ole'
 
-# Module for constants to be loaded int
+  # Module for constants to be loaded int
 
-module WORD_OLE_CONST
-end
+  module WORD_OLE_CONST
+  end
 
-class Word
-    
-  attr_reader :wd, :doc
+  class Word
 
-  def initialize( visible )
-    @wd = WIN32OLE.new('Word.Application')
+    attr_reader :wd, :doc
 
-    WIN32OLE.const_load(@wd, WORD_OLE_CONST) if WORD_OLE_CONST.constants.empty?
+    def initialize( visible )
+      @wd = WIN32OLE.new('Word.Application')
 
-    @wd.Visible = visible
-  end
+      WIN32OLE.const_load(@wd, WORD_OLE_CONST) if WORD_OLE_CONST.constants.empty?
 
-  def open(file)
-    @doc = @wd.Documents.Open(file)
-    @doc
-  end
+      @wd.Visible = visible
+    end
 
-  def save()
-    @doc.Save()
-    @doc
-  end
+    def open(file)
+      @doc = @wd.Documents.Open(file)
+      @doc
+    end
 
-  # Format : From WORD_OLE_CONST e.g WORD_OLE_CONST::WdFormatHTML
-  #
-  def save_as(name, format)
-    @doc.SaveAs(name, format)
-    return @doc
-  end
+    def save
+      @doc.Save()
+      @doc
+    end
 
-  # WdFormatFilteredHTML
-  # WdFormatHTML
-  def save_as_html(name)
-    @doc.SaveAs(name, WORD_OLE_CONST::WdFormatHTML)
-    return @doc
-  end
+    # Format : From WORD_OLE_CONST e.g WORD_OLE_CONST::WdFormatHTML
+    #
+    def save_as(name, format)
+      @doc.SaveAs(name, format)
+      @doc
+    end
 
-  def quit
-    @wd.quit()
+    # WdFormatFilteredHTML
+    # WdFormatHTML
+    def save_as_html(name)
+      @doc.SaveAs(name, WORD_OLE_CONST::WdFormatHTML)
+      @doc
+    end
+
+    def quit
+      @wd.quit
+    end
   end
-end
 
 else
 

data/lib/applications/ruby_poi_translations.rb

@@ -1,18 +1,19 @@
 # Copyright:: Autotelik Media Ltd
 # Author ::   Tom Statter
 # Date ::     July 2010
-# License::   
+# License::
 #
+# Translate Apache Poi Java data types/va;pues to Ruby world
 #
-if(DataShift::Guards::jruby?)
-  
+if DataShift::Guards.jruby?
+
   require 'java'
-  
+
   require 'benchmark'
-  require "poi-3.7-20101029.jar"
-  
+  require 'poi-3.7-20101029.jar'
+
   module RubyPoiTranslations
-  
+
     java_import 'org.apache.poi.poifs.filesystem.POIFSFileSystem'
 
     java_import 'org.apache.poi.hssf.usermodel.HSSFCell'
@@ -21,53 +22,54 @@ if(DataShift::Guards::jruby?)
     java_import 'org.apache.poi.hssf.usermodel.HSSFDataFormat'
     java_import 'org.apache.poi.hssf.usermodel.HSSFClientAnchor'
     java_import 'org.apache.poi.hssf.usermodel.HSSFRichTextString'
-    
+
     java_import 'org.apache.poi.hssf.util.HSSFColor'
 
     java_import 'java.io.ByteArrayOutputStream'
     java_import 'java.util.Date'
     java_import 'java.io.FileInputStream'
     java_import 'java.io.FileOutputStream'
-    
-    # Cell.CELL_TYPE_NUMERIC, Cell.CELL_TYPE_STRING, Cell.CELL_TYPE_FORMULA, 
+
+    # Cell.CELL_TYPE_NUMERIC, Cell.CELL_TYPE_STRING, Cell.CELL_TYPE_FORMULA,
     # Cell.CELL_TYPE_BLANK, Cell.CELL_TYPE_BOOLEAN, Cell.CELL_TYPE_ERROR
 
     def cell_value(cell)
-      case (cell.getCellType())
-      when HSSFCell::CELL_TYPE_FORMULA  then return cell.getCellFormula()
-      when HSSFCell::CELL_TYPE_NUMERIC  then return cell.getNumericCellValue()
-      when HSSFCell::CELL_TYPE_STRING   then return cell.getStringCellValue()
-      when HSSFCell::CELL_TYPE_BOOLEAN  then return cell.getBooleanCellValue()
-      when HSSFCell::CELL_TYPE_ERROR    then return cell.getErrorCellValue()
-
-      when HSSFCell::CELL_TYPE_BLANK    then return ""
+      return '' unless(cell)
+
+      case cell.getCellType
+      when HSSFCell::CELL_TYPE_FORMULA  then return cell.getCellFormula
+      when HSSFCell::CELL_TYPE_NUMERIC  then return cell.getNumericCellValue
+      when HSSFCell::CELL_TYPE_STRING   then return cell.getStringCellValue
+      when HSSFCell::CELL_TYPE_BOOLEAN  then return cell.getBooleanCellValue
+      when HSSFCell::CELL_TYPE_ERROR    then return cell.getErrorCellValue
+
+      when HSSFCell::CELL_TYPE_BLANK    then return ''
       end
     end
-    
-    
-    # Cell.CELL_TYPE_NUMERIC, Cell.CELL_TYPE_STRING, Cell.CELL_TYPE_FORMULA, 
+
+    # Cell.CELL_TYPE_NUMERIC, Cell.CELL_TYPE_STRING, Cell.CELL_TYPE_FORMULA,
     # Cell.CELL_TYPE_BLANK, Cell.CELL_TYPE_BOOLEAN, Cell.CELL_TYPE_ERROR
 
-    # TODO - properly
+    # TODO: - properly
     def poi_cell_value(data)
-      case(data)
+      case data
       when BigDecimal
         data.to_f
       when Numeric, TrueClass, FalseClass
-        data     
+        data
       else
         data.to_s
       end
     end
-    
-    # Return the suitable type for a HSSFCell from a Ruby data type           
+
+    # Return the suitable type for a HSSFCell from a Ruby data type
     def poi_cell_type(data)
-        
-      if(data.is_a?(Numeric))
+
+      if data.is_a?(Numeric)
         HSSFCell::CELL_TYPE_NUMERIC
-      elsif(data.nil?)
+      elsif data.nil?
         HSSFCell::CELL_TYPE_BLANK
-      elsif(data.is_a?(TrueClass) || data.is_a?(FalseClass))
+      elsif data.is_a?(TrueClass) || data.is_a?(FalseClass)
         HSSFCell::CELL_TYPE_BOOLEAN
       else
         HSSFCell::CELL_TYPE_STRING
@@ -75,5 +77,5 @@ if(DataShift::Guards::jruby?)
       # HSSFCell::CELL_TYPE_FORMULA
     end
   end
-  
-end
+
+end

data/lib/applications/spreadsheet_extensions.rb

@@ -1,31 +1,33 @@
 # Copyright:: Autotelik Media Ltd
 # Author ::   Tom Statter
 # Date ::     July 2010
-# License::   
+# License::
 #
 #
 # Extend the Sporeadsheet classes with some of our common methods
-# 
+#
 # ... to do extract into separate module with pure ruby that works with both POI and Spreadsheet
 
 require 'excel_base'
-  
-class Spreadsheet::Worksheet 
-     
-   include ExcelBase
-    
-  # Convert array into a header row
-  def set_headers(headers, apply_style = nil)
-    return if headers.empty?
-
-    headers.each_with_index do |datum, i|
-      self[0, i] = datum
+
+module Spreadsheet
+
+  class Worksheet
+
+    include DataShift::ExcelBase
+
+    # Convert array into a header row
+    def set_headers(headers, _apply_style = nil)
+      return if headers.empty?
+
+      headers.each_with_index do |header, i|
+        self[0, i] = header.to_s
+      end
     end
+
+    def num_rows
+      rows.size
+    end
+
   end
-  
-  def num_rows
-    rows.size
-  end
-  
 end
-  

data/lib/datashift.rb

@@ -1,6 +1,6 @@
-# Copyright:: (c) Autotelik Media Ltd 2010 - 2012 Tom Statter
+# Copyright:: (c) Autotelik Media Ltd 2010 - 2015 Tom Statter
 # Author ::   Tom Statter
-# Date ::     Aug 2010
+# Date ::     Aug 2015
 # License::   Free, Open Source.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
@@ -22,33 +22,16 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
-
-
-# Details::   Active Record Import/Export for .xls or CSV
-#
-# To pull DataShift commands into your main application :
-#
-#     require 'datashift'
-#
-#     DataShift::load_commands
 #
+require_relative 'datashift/guards'
+require_relative 'datashift/logging'
 
-module DataShift
+require 'active_support/core_ext/module/delegation'
 
-  def self.gem_version
-    unless(@gem_version)
-      if(File.exists?('VERSION'))
-        File.read( File.join('VERSION') ).match(/.*(\d+.\d+.\d+)/)
-        @gem_version = $1
-      else
-        @gem_version = '1.0.0'
-      end
-    end
-    @gem_version
-  end
+module DataShift
 
   def self.gem_name
-    "datashift"
+    'datashift'
   end
 
   def self.root_path
@@ -61,31 +44,55 @@ module DataShift
 
   def self.require_libraries
 
-    loader_libs = %w{ lib  }
+    loader_libs = %w(lib)
 
     # Base search paths - these will be searched recursively
     loader_paths = []
 
-    loader_libs.each {|l| loader_paths << File.join(root_path(), l) }
+    loader_libs.each { |l| loader_paths << File.join(root_path, l) }
 
     # Define require search paths, any dir in here will be added to LOAD_PATH
 
     loader_paths.each do |base|
-      $:.unshift base  if File.directory?(base)
+      $LOAD_PATH.unshift base if File.directory?(base)
       Dir[File.join(base, '**', '**')].each do |p|
-        if File.directory? p
-          $:.unshift p
-        end
+        $LOAD_PATH.unshift p if File.directory?(p)
       end
     end
 
-    require_libs = %w{ datashift loaders generators helpers }
+    begin
+      require_relative 'datashift/delimiters'
+      require_relative 'datashift/load_object'
+      require_relative 'generators/generator_base'
+      require_relative 'loaders/reporters/reporter'
+      require_relative 'loaders/loader_base'
+      require_relative 'exporters/exporter_base'
+    rescue => x
+      puts "Problem initializing gem #{x.inspect}"
+    end
+
+    require_libs = %w(
+      datashift/core_ext
+      datashift
+      datashift/mapping
+      datashift/model_methods
+      datashift/transformation
+      datashift/inbound_data
+      loaders
+      loaders/reporters
+      exporters
+      generators
+      helpers
+      applications
+    )
 
     require_libs.each do |base|
       Dir[File.join(library_path, base, '*.rb')].each do |rb|
-        unless File.directory? rb
-          #puts rb
-          require rb
+        # puts rb
+        begin
+          require_relative rb unless File.directory?(rb)
+        rescue => x
+          puts "Problem loading datashift file #{rb} - #{x.inspect}"
         end
       end
     end
@@ -101,7 +108,7 @@ module DataShift
 
   end
 
-  # Load all the datashift rake tasks and make them available throughout app
+  # Load all the datashift  tasks and make them available throughout app
   def self.load_tasks
     # Long parameter lists so ensure rake -T produces nice wide output
     ENV['RAKE_COLUMNS'] = '180'
@@ -109,45 +116,28 @@ module DataShift
     Dir["#{base}/*.rake"].sort.each { |ext| load ext }
   end
 
+  # Load all public datashift Thor commands and make them available throughout app
 
-  # Load all the datashift Thor commands and make them available throughout app
-
-  def self.load_commands()
-    base = File.join(library_path, 'thor', '**')
+  def self.load_commands
+    base = File.join(library_path, 'tasks')
 
     Dir["#{base}/*.thor"].each do |f|
       next unless File.file?(f)
-      Thor::Util.load_thorfile(f)
+      load(f)
     end
   end
 
 end
 
-
-require_relative 'datashift/delimiters'
-require_relative 'datashift/column_packer'
-require_relative 'datashift/logging'
-require_relative 'datashift/exceptions'
-require_relative 'datashift/guards'
-
-require_relative 'helpers/core_ext/to_b'
-require_relative 'helpers/core_ext/csv_file'
-
-require_relative 'datashift/method_detail'
-require_relative 'datashift/method_dictionary'
-require_relative 'datashift/method_mapper'
-require_relative 'datashift/model_mapper'
-
-DataShift::require_libraries
-
+DataShift.require_libraries
 
 module DataShift
-  if(Guards::jruby?)
+  if Guards.jruby?
     require 'java'
 
     class Object
       def add_to_classpath(path)
-        $CLASSPATH << File.join( DataShift.root_path, 'lib', path.gsub("\\", "/") )
+        $CLASSPATH << File.join( DataShift.root_path, 'lib', path.tr('\\', '/') )
       end
     end
   end

data/lib/datashift/binder.rb

@@ -0,0 +1,217 @@
+# Copyright:: (c) Autotelik Media Ltd 2015
+# Author ::   Tom Statter
+# License::   MIT
+#
+# Details::   Binds incoming string headers to domain model's attribute/association.
+#
+#             So a binding is a mapping between an Inbound Column and a ModelMethod
+#
+#             Example usage, load from a spreadsheet where the column names are only
+#             an approximation of the actual associations.
+#
+#             Given a column heading of 'Product Properties' on class Product,  map_inbound_headers would search
+#             the Product AR model, for a matching method or association, in this case  would
+#             bind this column, 'Product Properties', to the has_many association 'product_properties'.
+#
+#             This binding can be used to send spreadsheet row data to populate the product_properties on a product
+#
+#             Sometimes there may be no automatic binding available, but you may want to supply a custom method to process
+#             that column, in which case the following options may come into play
+#
+#             You can specify a list of columns to be bound even if no mapping is found using
+#               DataShift::Configuration.force_inclusion_of_columns = []
+#
+#             You can force inclusion of all columns despite whether mapping found or not using
+#               DataShift::Configuration.include_all = true
+#
+#
+module DataShift
+
+  class Binder
+
+    include DataShift::Logging
+
+    include DataShift::Delimiters
+    extend DataShift::Delimiters
+
+    attr_reader :mapped_class
+
+    attr_accessor :bindings, :missing_bindings
+
+    def initialize
+      reset
+    end
+
+    def reset
+      @bindings = []
+      @missing_bindings = []
+    end
+
+    def missing_bindings?
+      !missing_bindings.empty?
+    end
+
+    def headers_missing_bindings
+      missing_bindings.collect(&:inbound_name)
+    end
+
+    def indexes_missing_bindings
+      missing_bindings.collect(&:inbound_index)
+    end
+
+    def forced_inclusion_columns
+      [*DataShift::Configuration.call.force_inclusion_of_columns]
+    end
+
+    def forced
+      forced_inclusion_columns.compact.collect { |f| f.to_s.downcase }
+    end
+
+    def forced?(column_name)
+      forced.include?(column_name.to_s.downcase)
+    end
+
+    def include_all?
+      DataShift::Configuration.call.include_all_columns == true
+    end
+
+    # Build complete picture of the methods whose names listed in columns
+    # Handles method names as defined by a user, from spreadsheets or file headers where the names
+    # specified may not be exactly as required e.g handles capitalisation, white space, _ etc
+    #
+    # The header can also contain the fields to use in lookups, separated with Delimiters ::column_delim
+    # For example specify that lookups on has_one association called 'product', be performed using name'
+    #   product:name
+    #
+    # The header can also contain a default value for the lookup field, again separated with Delimiters ::column_delim
+    #
+    # For example specify lookups on assoc called 'user', be performed using 'email' == 'test@blah.com'
+    #
+    #   user:email:test@blah.com
+    #
+    # Returns: Array of matching method_details, including nils for non matched items
+    #
+    # N.B Columns that could not be mapped are left in the array as NIL
+    #
+    # This is to support clients that need to map via the index on @method_details
+    #
+    # Other callers can simply call compact on the results if the index not important.
+    #
+    # The MethodDetails instance will contain a pointer to the column index from which it was mapped.
+    #
+    def map_inbound_headers(klass, columns)
+
+      @mapped_class = klass
+
+      # If klass not in Dictionary yet, add to dictionary all possible operators on klass
+      # which can be used to map headers and populate an object of type klass
+      model_method_mgr = ModelMethods::Manager.catalog_class(klass)
+
+      reset
+
+      [*columns].each_with_index do |col_data, col_index|
+        raw_col_data = col_data.to_s
+
+        if raw_col_data.nil? || raw_col_data.empty?
+          logger.warn("Column list contains empty or null header at index #{col_index}")
+          bindings << NoMethodBinding.new(raw_col_data, col_index)
+          next
+        end
+
+        # Header DSL Name::Where::Value:Misc
+
+        raw_col_name, where_field, where_value, *data = raw_col_data.split(column_delim)
+
+        # Find the domain model method details
+        model_method = model_method_mgr.search(raw_col_name)
+
+        # if not found via raw name, try various alternatives
+
+        if( model_method.nil? && (include_all? || forced?(raw_col_name)) )
+          logger.debug("Operator #{raw_col_name} not found but forced inclusion set - adding as :method")
+          model_method = model_method_mgr.insert(raw_col_name, :method)
+        end
+
+        unless model_method
+          Binder.substitutions(raw_col_name).each do |n|
+            model_method = model_method_mgr.search(n)
+            break if model_method
+          end
+        end
+
+        if(model_method)
+
+          binding = MethodBinding.new(raw_col_name, col_index, model_method)
+
+          # we slurped up all possible data in split, turn it back into original string
+          binding.add_column_data(data.join(column_delim))
+
+          if where_field
+            logger.info("Lookup query field [#{where_field}] - specified for association #{model_method.operator}")
+
+            begin
+              binding.add_lookup(model_method, where_field, where_value)
+            rescue
+              add_missing(raw_col_data, col_index,
+                          "Field [#{where_field}] Not Found for [#{raw_col_name}] (#{model_method.operator})")
+              next
+            end
+
+          end
+
+          logger.debug("Column [#{col_data}] (#{col_index}) - mapped to :\n#{model_method.pp}")
+
+          bindings << binding
+
+        else
+          logger.debug("No operator or association found for Header [#{raw_col_name}]")
+
+          add_missing(raw_col_data, col_index, "No operator or association found for Header [#{raw_col_name}]")
+        end
+      end
+
+      bindings
+    end
+
+    # Essentially we map any string collection of field names, not just headers from files
+    alias map_inbound_fields map_inbound_headers
+
+    def add_missing(col_data, col_index, reason)
+      logger.warn(reason)
+
+      missing = NoMethodBinding.new(col_data, col_index, reason: reason)
+
+      missing_bindings << missing
+      bindings << missing
+    end
+
+    # TODO: - check out regexp to do this work better plus Inflections ??
+    # Want to be able to handle any of ["Count On hand", 'count_on_hand', "Count OnHand", "COUNT ONHand" etc]
+    def self.substitutions(external_name)
+      name = external_name.to_s
+
+      [
+        name.downcase,
+        name.tableize,
+        name.tr(' ', '_'),
+        name.tr(' ', '_').downcase,
+        name.gsub(/(\s+)/, '_').downcase,
+        name.delete(' '),
+        name.delete(' ').downcase,
+        name.tr(' ', '_').underscore
+      ]
+    end
+
+    # The raw client supplied names
+    def method_names
+      bindings.collect( &:inbound_name )
+    end
+
+    # The true operator names discovered from model
+    def operator_names
+      bindings.collect( &:operator )
+    end
+
+  end
+
+end