datashift 0.40.3 → 0.40.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 53ca961606b77228304650d8087f155975c54691f7b7dd8b9c7aa17595c3088f
-  data.tar.gz: 01bb10cbbfac38feb2ec994a5db033aba22cc423e1051c42c0fa1e1c403910aa
+SHA1:
+  metadata.gz: d7be9f9beb4237a952c953f59c0e887124a3222a
+  data.tar.gz: e68197e5dd89fa5a21165b9bd8e606e58650dfc7
 SHA512:
-  metadata.gz: aa5e57594b2b5f8e893b2153300566a06f298d0174059a963d1de3ba1b797e960066a0b5c6949397b4b335f58978b3f5cfea460bab39bcaa776fc1b6a32a6b3d
-  data.tar.gz: 3677cc69f278063588621cdd7849a4633cc755cb03deb9d52f748be2ade59575eb4160fc1177e612ac331a23e2edd57f5bac7f37b50b80bb0a9954ff99da2051
+  metadata.gz: d888089a643a05aedb27deabb01c814dc27940d1ef96475ebb320f6b71799629ae89797df740217569dc9a86f5abe8010ca4e6f5be5fb077b97594f5d49fed37
+  data.tar.gz: dfc8a6dca16c663df3b0ae8959de6382f0ff10feeadaeaa019e04083d37ccc3512fcedc4deee610f3115ceb43c6d6bef152a80e33f7afac5e76cd38d2b8f4a95

data/README.md

@@ -249,9 +249,14 @@ is to run our rail's install generator :
 ```ruby
 rails g datashift:install
 ```
+
+You can create a model specific file at anytime via
+```bash
+    thor datashift:config:generate:import
+```
  
-To create such a file manually, create an initialisation file within `config/initializers`, 
-and see `lib/datashift/configuration.rb` for details of options, for example
+To create a Rails tyle config block manually, create an initialisation file within `config/initializers`, 
+and see [lib/datashift/configuration.rb](lib/datashift/configuration.rb`) for details of all possible options, for example
 
 ```ruby
 DataShift::Configuration.call do |c|

data/datashift.thor

@@ -10,7 +10,7 @@
 #             type applications, files and databases.
 #
 #             Provides support for moving data between .xls (Excel/OpenOffice)
-#             Spreedsheets via Ruby and AR, enabling direct import/export of 
+#             Spreedsheets via Ruby and AR, enabling direct import/export of
 #             ActiveRecord models with all their associations from database.
 #
 #             Provides support for moving data between csv files and AR, enabling direct
@@ -18,14 +18,14 @@
 #
 require 'thor'
 
-$:.push File.expand_path("lib", __FILE__)
+$LOAD_PATH.push File.expand_path('lib')
 
 require 'datashift'
 require 'factory_bot_rails'
 require 'database_cleaner'
 
 require_relative File.join('spec', 'support/sandbox')
-require_relative File.join('spec' ,'support/datashift_test_helpers')
+require_relative File.join('spec', 'support/datashift_test_helpers')
 
 module Datashift
 
@@ -33,7 +33,7 @@ module Datashift
 
     include DataShift::TestHelpers
 
-    desc "lint", "Run in spec - Verify that FactoryBot factories are valid"
+    desc 'lint', 'Run in spec - Verify that FactoryBot factories are valid'
 
     def lint
 
@@ -44,7 +44,7 @@ module Datashift
       begin
         DatabaseCleaner.start
 
-        puts "Running FactoryBot.lint"
+        puts 'Running FactoryBot.lint'
         FactoryBot.lint
       ensure
         DatabaseCleaner.clean
@@ -52,36 +52,41 @@ module Datashift
 
     end
 
-
-    desc "sandbox", 'Rebuild the dummy rails app in spec - required for testing'
+    desc 'sandbox', 'Rebuild the dummy rails app in spec - required for testing'
 
     def sandbox
       # Need an active record DB to test against, so we manage own Rails sandbox
       DataShift::Sandbox.gen_rails_sandbox( :force )
     end
 
-    desc "build", 'Build gem and install in one step'
+    desc 'build', 'Build gem and install in one step'
 
-    method_option :bump, :aliases => '-b', type: :string, desc:  "Bump the version", required: false
+    method_option :bump, aliases: '-b', type: :string, desc: 'Bump the version', required: false
 
-    method_option :push, :aliases => '-p', :desc => "Push resulting gem to rubygems.org"
+    method_option :push, aliases: '-p', desc: 'Push resulting gem to rubygems.org'
 
-    method_option :install, :aliases => '-i',
-                  :desc => "Install freshly built gem locally", type: :boolean, default: false
+    method_option :install, aliases: '-i',
+                            desc: 'Install freshly built gem locally', type: :boolean, default: false
 
     def build
 
-      raise "Please bump to a new version to install at rubygems" if options[:install] && options[:bump].blank?
-      version = options[:bump]
+      if options[:push] && (options[:bump].blank? || options[:bump] !~ /^(\d+\.)?(\d+\.)?(\*|\d+)$/)
+        puts 'ERROR: Please bump to a new numeric version to push to rubygems'
+        exit(-1)
+      end
+
+      version = options[:bump] || DataShift::VERSION
 
       # Bump the VERSION file in library
-      File.open( File.join('lib/datashift/version.rb'), 'w') do |f|
-        f << "module DataShift\n"
-        f << "    VERSION = '#{version}'.freeze unless defined?(VERSION)\n"
-        f << "end\n"
-      end if(options[:bump].present?)
+      if options[:bump].present?
+        File.open( File.join('lib/datashift/version.rb'), 'w') do |f|
+          f << "module DataShift\n"
+          f << "    VERSION = '#{version}'.freeze unless defined?(VERSION)\n"
+          f << "end\n"
+        end
+      end
 
-      build_cmd =  "gem build datashift.gemspec"
+      build_cmd = 'gem build datashift.gemspec'
 
       puts "\n*** Running build cmd [#{build_cmd}]"
 
@@ -111,13 +116,13 @@ module Datashift
         if File.exist?(env)
           begin
             require env
-          rescue => e
+          rescue StandardError => e
             logger.error("Failed to initialise ActiveRecord : #{e.message}")
-            raise ConnectionError.new("Failed to initialise ActiveRecord : #{e.message}")
+            raise ConnectionError, "Failed to initialise ActiveRecord : #{e.message}"
           end
 
         else
-          raise DataShift::PathError.new('No config/environment.rb found - cannot initialise ActiveRecord')
+          raise DataShift::PathError, 'No config/environment.rb found - cannot initialise ActiveRecord'
         end
       end
     end

data/lib/datashift.rb

@@ -47,7 +47,7 @@ module DataShift
 
   def self.require_libraries
 
-    loader_libs = %w(lib)
+    loader_libs = %w[lib]
 
     # Base search paths - these will be searched recursively
     loader_paths = []
@@ -70,11 +70,11 @@ module DataShift
       require_relative 'datashift/loaders/reporters/reporter'
       require_relative 'datashift/loaders/loader_base'
       require_relative 'datashift/exporters/exporter_base'
-    rescue => x
+    rescue StandardError => x
       puts "Problem initializing gem #{x.inspect}"
     end
 
-    require_libs = %w(
+    require_libs = %w[
       datashift/core_ext
       datashift
       datashift/mapping
@@ -88,14 +88,14 @@ module DataShift
       datashift/helpers
       datashift/applications
       datashift/populators
-    )
+    ]
 
     require_libs.each do |base|
       Dir[File.join(library_path, base, '*.rb')].each do |rb|
         # puts rb
         begin
           require_relative rb unless File.directory?(rb)
-        rescue => x
+        rescue StandardError => x
           puts "WARNING - Problem loading datashift file #{rb} - #{x.inspect}"
           puts x.backtrace
         end
@@ -128,7 +128,7 @@ module DataShift
 
     Dir["#{base}/*.thor"].each do |f|
       next unless File.file?(f)
-      load(f)
+      Thor::Util.load_thorfile(f)
     end
   end
 

data/lib/datashift/binder.rb

@@ -101,14 +101,16 @@ module DataShift
 
       # If klass not in Dictionary yet, add to dictionary all possible operators on klass
       # which can be used to map headers and populate an object of type klass
-      model_method_mgr = ModelMethods::Manager.catalog_class(klass)
+      model_methods_collection = ModelMethods::Manager.catalog_class(klass)
+
+      bound = bindings.map(&:source)
 
       [*columns].each_with_index do |col_data, col_index|
         raw_col_data = col_data.to_s.strip
 
         if raw_col_data.nil? || raw_col_data.empty?
           logger.warn("Column list contains empty or null header at index #{col_index}")
-          bindings << NoMethodBinding.new(raw_col_data, col_index)
+          bindings << NoMethodBinding.new(raw_col_data, idx: col_index)
           next
         end
 
@@ -119,25 +121,38 @@ module DataShift
         #
         raw_col_name, where_field, where_value, *data = raw_col_data.split(column_delim).map(&:strip)
 
+        # Config loaded details trump internal mappings. User may not bother setting index of the column
+        # in config, so attempt now to match it to actual header
+        if bound.include?(raw_col_name)
+          external = bindings.find { |b| b.source == raw_col_name }
+          external.index = col_index if(external && external.index.nil?)
+          next
+        end
+
         # Find the domain model method details
-        model_method = model_method_mgr.search(raw_col_name)
+        model_method = model_methods_collection.search(raw_col_name)
 
-        # No such column, but if config set to include it, for example for delegated methods, add as op type :assignment
+        # No such column, so check config
+        #
+        #   Forced inclusion for example for delegated methods that do not show up in reflection.
+        #
+        # Add as operator type :assignment
+        #
         if( model_method.nil? && (include_all? || forced?(raw_col_name)) )
           logger.debug("Operator #{raw_col_name} not found but forced inclusion set - adding as :assignment")
-          model_method = model_method_mgr.insert(raw_col_name, :assignment)
+          model_method = model_methods_collection.insert(raw_col_name, :assignment)
         end
 
         unless model_method
           Binder.substitutions(raw_col_name).each do |n|
-            model_method = model_method_mgr.search(n)
+            model_method = model_methods_collection.search(n)
             break if model_method
           end
         end
 
         if(model_method)
 
-          binding = MethodBinding.new(raw_col_name, col_index, model_method)
+          binding = MethodBinding.new(raw_col_name, model_method, idx: col_index)
 
           # we slurped up all possible data in split, turn it back into original string
           binding.add_column_data(data.join(column_delim))
@@ -147,7 +162,7 @@ module DataShift
 
             begin
               binding.add_lookup(model_method, where_field, where_value)
-            rescue => e
+            rescue StandardError => e
               logger.error(e.message)
               add_missing(raw_col_data, col_index, "Field [#{where_field}] Not Found for [#{raw_col_name}] (#{model_method.operator})")
               next
@@ -167,8 +182,7 @@ module DataShift
     end
 
     def add_bindings_from_nodes( nodes )
-      logger.debug("Adding  [#{nodes.size}] custom bindings")
-      nodes.each { |n| bindings << n.method_binding }
+      nodes.each { |n| bindings << n.method_binding unless n.is_a?(NoMethodBinding) }
     end
 
     # Essentially we map any string collection of field names, not just headers from files
@@ -177,7 +191,7 @@ module DataShift
     def add_missing(col_data, col_index, reason)
       logger.warn(reason)
 
-      missing = NoMethodBinding.new(col_data, col_index, reason: reason)
+      missing = NoMethodBinding.new(col_data, reason: reason, idx: col_index)
 
       missing_bindings << missing
       bindings << missing
@@ -210,6 +224,11 @@ module DataShift
       bindings.collect( &:operator )
     end
 
+    # Find a binding, matches raw client supplied names e.g header and has a valid index
+    def find_for_source( name )
+      bindings.find { |b| b.source == name && b.index }
+    end
+
   end
 
 end

data/lib/datashift/configuration.rb

@@ -11,6 +11,14 @@ module DataShift
 
   class Configuration
 
+
+    def self.parse_yaml( yaml_file )
+      bound_template = ERB.new( IO.read(yaml_file)).result
+
+      YAML.safe_load(bound_template, [Date, Time, Symbol] )
+    end
+
+
     # List of association +TYPES+ to INCLUDE [:assignment, :enum, :belongs_to, :has_one, :has_many, :method]
     # Defaults to [:assignment, :enum]
     #
@@ -122,11 +130,11 @@ module DataShift
     attr_accessor :image_path_prefix
 
     def self.rails_columns
-      @rails_standard_columns ||= [:id, :created_at, :created_on, :updated_at, :updated_on]
+      @rails_standard_columns ||= %i[id created_at created_on updated_at updated_on]
     end
 
     def initialize
-      @with = [:assignment, :enum]
+      @with = %i[assignment enum]
       @exclude = []
       @remove_columns = []
 

data/lib/datashift/core_ext/array.rb

@@ -1,12 +1,12 @@
 Array.class_eval do
 
-  ARRAY_FWDABLE_EXCLUDED_METHODS = [
-    :class, :singleton_class, :clone, :dup, :initialize_dup, :initialize_clone,
-    :freeze, :methods, :singleton_methods, :protected_methods, :private_methods, :public_methods,
-    :instance_variables, :instance_variable_get, :instance_variable_set, :instance_variable_defined?,
-    :instance_of?, :kind_of?, :is_a?, :tap, :send, :public_send, :respond_to?, :respond_to_missing?,
-    :extend, :display, :method, :public_method, :define_singleton_method, :object_id, :equal?,
-    :instance_eval, :instance_exec, :__send__, :__id__
+  ARRAY_FWDABLE_EXCLUDED_METHODS = %i[
+    class singleton_class clone dup initialize_dup initialize_clone
+    freeze methods singleton_methods protected_methods private_methods public_methods
+    instance_variables instance_variable_get instance_variable_set instance_variable_defined?
+    instance_of? kind_of? is_a? tap send public_send respond_to? respond_to_missing?
+    extend display method public_method define_singleton_method object_id equal?
+    instance_eval instance_exec __send__ __id__
   ].freeze
 
   def self.delegated_methods_for_fwdable

data/lib/datashift/delimiters.rb

@@ -72,7 +72,7 @@ module DataShift
     # Objects can be created with multiple facets in single columns.
     # In this example a single Product can be configured with a consolidated mime and print types
     #
-    # mime_type:jpeg,PDF ; print_type:colour	 equivalent to
+    # mime_type:jpeg,PDF ; print_type:colour   equivalent to
     #
     #   => mime_type:jpeg;print_type:colour | mime_type:PDF; print_type:colour
 

data/lib/datashift/doc_context.rb

@@ -119,7 +119,7 @@ module DataShift
       logger.debug("SAVING #{load_object.class} : #{load_object.inspect}")
       begin
         load_object.save!
-      rescue => e
+      rescue StandardError => e
         logger.error( "Save Error : #{e.inspect} on #{load_object.class}")
         logger.error(e.backtrace)
         false

data/lib/datashift/excel_base.rb

@@ -105,7 +105,7 @@ module DataShift
 
     # Pass a set of AR records
     def ar_to_xls(records, start_row: 1, headers: nil, data_flow_schema: nil)
-      return if (!exportable?(records.first) || records.empty?)
+      return if !exportable?(records.first) || records.empty?
 
       # assume header row present
       row_index = start_row
@@ -140,7 +140,7 @@ module DataShift
     def ar_to_xls_cell(row_idx, col_idx, record, ar_method)
       datum = record.send(ar_method)
       self[row_idx, col_idx] = datum
-    rescue => e
+    rescue StandardError => e
       logger.error("Failed to export #{datum} from #{ar_method.inspect} to column #{col_idx}")
       logger.error(e.message)
       logger.error(e.backtrace)

data/lib/datashift/exporters/csv_exporter.rb

@@ -100,7 +100,6 @@ module DataShift
           csv.add_row(row)
         end
       end
-
     ensure
       DataShift::Configuration.call.with = state
     end # end write file

data/lib/datashift/exporters/excel_exporter.rb

@@ -43,7 +43,7 @@ module DataShift
     def export(file, export_records, options = {})
       records = [*export_records]
 
-      if(records.nil? || records.empty?)
+      if records.blank?
         logger.warn('Excel Export - No objects supplied for export - no file written')
         return
       end
@@ -108,7 +108,7 @@ module DataShift
 
       logger.info("Processing [#{records.size}] #{klass} records to Excel")
 
-      # TODO - prepare_data_flow_schema here in middle of export, plus reaching through nodes to klass, does not smell right
+      # TODO: - prepare_data_flow_schema here in middle of export, plus reaching through nodes to klass, does not smell right
       prepare_data_flow_schema(klass) unless @data_flow_schema && @data_flow_schema.nodes.klass == klass
 
       export_headers(klass)
@@ -136,7 +136,7 @@ module DataShift
             else
               excel[row, column] = obj.send( model_method.operator )
             end
-          rescue => x
+          rescue StandardError => x
             logger.error("Failed to write #{model_method.inspect} to Excel")
             logger.error(x.inspect)
           end
@@ -149,7 +149,6 @@ module DataShift
 
       logger.info("Writing Excel to file [#{file_name}]")
       excel.write( file_name )
-
     ensure
       DataShift::Configuration.call.with = state
 

data/lib/datashift/file_definitions.rb

@@ -136,9 +136,7 @@ module FileDefinitions
     #
     def create_fixed_definition( field_range_map )
 
-      unless field_range_map.is_a?(Hash)
-        raise ArgumentError, 'Please supply hash to create_fixed_definition'
-      end
+      raise ArgumentError, 'Please supply hash to create_fixed_definition' unless field_range_map.is_a?(Hash)
 
       keys = field_range_map.keys.collect(&:to_s)
       string_map = Hash[*keys.zip(field_range_map.values).flatten]

data/lib/datashift/inbound_data/method_binding.rb

@@ -22,7 +22,7 @@ module DataShift
     attr_reader :inbound_column
 
     delegate :source, to: :inbound_column, allow_nil: true
-    delegate :index, to: :inbound_column, allow_nil: true
+    delegate :index, 'index=', to: :inbound_column, allow_nil: true
 
     # Is this method detail a valid mapping, aids identifying unmapped/unmappable columns
     attr_accessor :valid
@@ -36,7 +36,7 @@ module DataShift
     #
     # col_types can typically be derived from klass.columns - set of ActiveRecord::ConnectionAdapters::Column
 
-    def initialize(name, idx, model_method)
+    def initialize(name, model_method, idx: nil)
       @inbound_column = InboundData::Column.new(name, idx)
 
       @model_method = model_method
@@ -130,10 +130,10 @@ module DataShift
 
     attr_accessor :reason
 
-    def initialize(client_name = '', client_idx = -1, options = {})
-      super(client_name, client_idx, nil)
+    def initialize(client_name = '', reason: nil, idx: 1 )
+      super(client_name, nil, idx: idx)
 
-      @reason = options[:reason] || ''
+      @reason = reason || ''
     end
 
     def invalid?