datashift 0.10.1 → 0.10.2

data/Rakefile

@@ -22,7 +22,12 @@ require 'rubygems'
 
 require 'rake'
 
-require 'lib/datashift'
+lib = File.expand_path('../lib/', __FILE__)
+
+$:.unshift '.' 
+$:.unshift lib unless $:.include?(lib)
+
+require 'datashift'
 
 require 'jeweler'
 Jeweler::Tasks.new do |gem|

data/VERSION

@@ -1 +1 @@
-0.10.1
+0.10.2

data/datashift.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "datashift"
-  s.version = "0.10.1"
+  s.version = "0.10.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Thomas Statter"]
-  s.date = "2012-09-25"
+  s.date = "2012-10-21"
   s.description = "Comprehensive tools to import/export between Excel/CSV and ActiveRecord databases, Rails apps, and any Ruby project."
   s.email = "rubygems@autotelik.co.uk"
   s.extra_rdoc_files = [
@@ -46,6 +46,7 @@ Gem::Specification.new do |s|
     "lib/datashift/method_mapper.rb",
     "lib/datashift/model_mapper.rb",
     "lib/datashift/populator.rb",
+    "lib/datashift/querying.rb",
     "lib/exporters/csv_exporter.rb",
     "lib/exporters/excel_exporter.rb",
     "lib/exporters/exporter_base.rb",
@@ -54,7 +55,6 @@ Gem::Specification.new do |s|
     "lib/generators/generator_base.rb",
     "lib/guards.rb",
     "lib/helpers/core_ext/to_b.rb",
-    "lib/helpers/rake_utils.rb",
     "lib/java/poi-3.7/._poi-3.7-20101029.jar5645100390082102460.tmp",
     "lib/java/poi-3.7/LICENSE",
     "lib/java/poi-3.7/NOTICE",
@@ -82,6 +82,11 @@ Gem::Specification.new do |s|
     "lib/thor/paperclip.thor",
     "lib/thor/tools.thor",
     "spec/Gemfile",
+    "spec/MissingAttachmentRecords/DEMO_001_ror_bag.jpeg",
+    "spec/MissingAttachmentRecords/DEMO_002_Powerstation.jpeg",
+    "spec/MissingAttachmentRecords/DEMO_002_Powerstation.jpg",
+    "spec/MissingAttachmentRecords/DEMO_003_ror_mug.jpeg",
+    "spec/MissingAttachmentRecords/DEMO_004_ror_ringer.jpeg",
     "spec/csv_exporter_spec.rb",
     "spec/csv_loader_spec.rb",
     "spec/datashift_spec.rb",
@@ -100,19 +105,20 @@ Gem::Specification.new do |s|
     "spec/fixtures/config/database.yml",
     "spec/fixtures/db/datashift_test_models_db.sqlite",
     "spec/fixtures/db/migrate/20110803201325_create_test_bed.rb",
+    "spec/fixtures/db/migrate/20121009161700_add_digitals.rb",
     "spec/fixtures/images/DEMO_001_ror_bag.jpeg",
-    "spec/fixtures/images/DEMO_002_Powerstation.jpg",
+    "spec/fixtures/images/DEMO_002_Powerstation.jpeg",
     "spec/fixtures/images/DEMO_003_ror_mug.jpeg",
     "spec/fixtures/images/DEMO_004_ror_ringer.jpeg",
     "spec/fixtures/load_datashift.thor",
     "spec/fixtures/models/category.rb",
+    "spec/fixtures/models/digital.rb",
     "spec/fixtures/models/empty.rb",
     "spec/fixtures/models/loader_release.rb",
     "spec/fixtures/models/long_and_complex_table_linked_to_version.rb",
     "spec/fixtures/models/milestone.rb",
     "spec/fixtures/models/owner.rb",
     "spec/fixtures/models/project.rb",
-    "spec/fixtures/models/test_model_defs.rb",
     "spec/fixtures/models/version.rb",
     "spec/fixtures/simple_export_spec.xls",
     "spec/fixtures/simple_template_spec.xls",
@@ -120,6 +126,7 @@ Gem::Specification.new do |s|
     "spec/loader_spec.rb",
     "spec/method_dictionary_spec.rb",
     "spec/method_mapper_spec.rb",
+    "spec/paperclip_loader_spec.rb",
     "spec/rails_sandbox/.gitignore",
     "spec/rails_sandbox/Gemfile",
     "spec/rails_sandbox/README.rdoc",
@@ -190,7 +197,7 @@ Gem::Specification.new do |s|
   s.homepage = "http://github.com/autotelik/datashift"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.15"
+  s.rubygems_version = "1.8.24"
   s.summary = "Shift data betwen Excel/CSV and any Ruby app"
 
   if s.respond_to? :specification_version then

data/lib/datashift.rb

@@ -33,27 +33,9 @@
 #     DataShift::load_commands
 #
 require 'rbconfig'
-  
-module DataShift
-
-  module Guards
-
-    def self.jruby?
-      return RUBY_PLATFORM == "java"
-    end
-    def self.mac?
-      RbConfig::CONFIG['target_os'] =~ /darwin/i
-    end
+require 'guards'
 
-    def self.linux?
-      RbConfig::CONFIG['target_os'] =~ /linux/i
-    end
-
-    def self.windows?
-      RbConfig::CONFIG['target_os'] =~ /mswin|mingw/i
-    end
-
-  end
+module DataShift
 
   if(Guards::jruby?)
     require 'java'

data/lib/datashift/exceptions.rb

@@ -10,4 +10,6 @@ module DataShift
   class MissingHeadersError < StandardError; end
   class MissingMandatoryError < StandardError; end
 
+  class RecordNotFound < StandardError; end
+  
 end

data/lib/datashift/method_detail.rb

@@ -12,6 +12,7 @@
 #
 require 'to_b'
 require 'logging'
+require 'populator'
 require 'set'
 
 module DataShift
@@ -20,6 +21,9 @@ module DataShift
 
     include DataShift::Logging
     
+    include DataShift::Populator
+    extend DataShift::Populator
+    
     def self.supported_types_enum
       @type_enum ||= Set[:assignment, :belongs_to, :has_one, :has_many]
       @type_enum
@@ -35,13 +39,17 @@ module DataShift
     @@insistent_find_by_list ||= [:name, :title, :id]
 
     # Name is the raw, client supplied name
-    attr_reader :name, :col_type, :current_value
+    attr_accessor :name
+    attr_accessor :column_index
+  
+    # The rel col type from the DB
+    attr_reader :col_type, :current_value
 
     attr_reader :operator, :operator_type
 
     # TODO make it a list/primary keys
     attr_accessor :find_by_operator, :find_by_value
-    
+        
     # Store the raw (client supplied) name against the active record  klass(model).
     # Operator is the associated method call on klass,
     # so client name maybe Price but true operator is price
@@ -68,6 +76,8 @@ module DataShift
       else
         @col_type = col_types[operator]
       end
+      
+      @column_index = -1
     end
 
 
@@ -163,12 +173,6 @@ module DataShift
       "#{@name} => #{operator}"
     end
 
-
-    def self.insistent_method_list
-      @insistent_method_list ||= [:to_s, :to_i, :to_f, :to_b]
-      @insistent_method_list
-    end
-
     private
 
     # Attempt to find the associated object via id, name, title ....
@@ -188,7 +192,7 @@ module DataShift
             end
           rescue => e
             puts "ERROR: #{e.inspect}"
-            if(x == MethodDetail::insistent_method_list.last)
+            if(x == Populator::insistent_method_list.last)
               raise "I'm sorry I have failed to assign [#{value}] to #{@assignment}" unless value.nil?
             end
           end
@@ -211,7 +215,7 @@ module DataShift
             end
           rescue => e
             puts "ERROR: #{e.inspect}"
-            if(x == MethodDetail::insistent_method_list.last)
+            if(x == Populator::insistent_method_list.last)
               raise "I'm sorry I have failed to assign [#{value}] to #{operator}" unless value.nil?
             end
           end
@@ -220,25 +224,7 @@ module DataShift
     end
 
     def insistent_assignment( record, value )
-      #puts "DEBUG: RECORD CLASS #{record.class}"
-      op = operator + '='
-    
-      begin
-        record.send(op, value)
-      rescue => e
-        MethodDetail::insistent_method_list.each do |f|
-          begin
-            record.send(op, value.send( f) )
-            break
-          rescue => e
-            #puts "DEBUG: insistent_assignment: #{e.inspect}"
-            if f == MethodDetail::insistent_method_list.last
-              puts  "I'm sorry I have failed to assign [#{value}] to #{operator}"
-              raise "I'm sorry I have failed to assign [#{value}] to #{operator}" unless value.nil?
-            end
-          end
-        end
-      end
+      Populator::insistent_assignment( record, value, operator)
     end
     
     private

data/lib/datashift/method_dictionary.rb

@@ -16,7 +16,6 @@ module DataShift
     def initialize
     end
 
-
     # Has the dictionary been populated for  klass
     def self.for?(klass)
       return !(has_many[klass] || belongs_to[klass] || has_one[klass] || assignments[klass]).nil?
@@ -122,22 +121,12 @@ module DataShift
       method_details_mgrs[klass] = method_details_mgr
       
     end
-        
-    # Find the proper format of name, appropriate call + column type for a given name.
-    # e.g Given users entry in spread sheet check for pluralization, missing underscores etc
-    #
-    # If not nil, returned method can be used directly in for example klass.new.send( call, .... )
-    #
-    def self.find_method_detail( klass, external_name )
-
-      method_details_mgr = get_method_details_mgr( klass )
-         
-      # md_mgr.all_available_operators.each { |l| puts "DEBUG: Mapped Method : #{l.inspect}" }
-        
+   
+    # TODO - check out regexp to do this work better plus Inflections ??
+    # Want to be able to handle any of ["Count On hand", 'count_on_hand', "Count OnHand", "COUNT ONHand" etc]
+    def self.substitutions(external_name)
       name = external_name.to_s
-     
-      # TODO - check out regexp to do this work better plus Inflections ??
-      # Want to be able to handle any of ["Count On hand", 'count_on_hand', "Count OnHand", "COUNT ONHand" etc]
+      
       [
         name,
         name.tableize,
@@ -146,20 +135,46 @@ module DataShift
         name.gsub(/(\s+)/, '_').downcase,
         name.gsub(' ', ''),
         name.gsub(' ', '').downcase,
-        name.gsub(' ', '_').underscore].each do |n|
-      
-        # Try each association type, returning first that contains matching operator with name n
+        name.gsub(' ', '_').underscore
+      ]
+    end
+    
+    # Find the proper format of name, appropriate call + column type for a given name.
+    # e.g Given users entry in spread sheet check for pluralization, missing underscores etc
+    #
+    # If not nil, returned method can be used directly in for example klass.new.send( call, .... )
+    #
+    def self.find_method_detail( klass, external_name )
+
+      method_details_mgr = get_method_details_mgr( klass )
+         
+      # md_mgr.all_available_operators.each { |l| puts "DEBUG: Mapped Method : #{l.inspect}" }      
+      substitutions(external_name).each do |n|
       
+        # Try each association type, returning first that contains matching operator with name n    
         MethodDetail::supported_types_enum.each do |t|
           method_detail = method_details_mgr.find(n, t)
           return method_detail.clone if(method_detail)
-        end
-        
+        end  
       end
 
       nil
     end
+    
+    # Assignments can contain things like delegated methods, this returns a matching 
+    # method details only when a true database column   
+    def self.find_method_detail_if_column( klass, external_name )
 
+      method_details_mgr = get_method_details_mgr( klass )
+      
+      substitutions(external_name).each do |n|
+        method_detail = method_details_mgr.find(n, :assignment)
+        return method_detail if(method_detail && method_detail.col_type) 
+      end
+      
+      nil
+    end
+    
     def self.clear
       belongs_to.clear
       has_many.clear

data/lib/datashift/method_mapper.rb

@@ -24,7 +24,6 @@ module DataShift
 
     include DataShift::Logging
     
-    attr_accessor :header_row, :headers
     attr_accessor :method_details, :missing_methods
   
     
@@ -45,68 +44,109 @@ module DataShift
   
     def initialize
       @method_details = []
-      @headers = []
     end
 
     # Build complete picture of the methods whose names listed in columns
     # Handles method names as defined by a user, from spreadsheets or file headers where the names
     # specified may not be exactly as required e.g handles capitalisation, white space, _ etc
-    # Returns: Array of matching method_details
+    # 
+    # The header can also contain the fields to use in lookups, separated with MethodMapper::column_delim
+    # 
+    #   product:name or project:title  or user:email
+    # 
+    # Returns: Array of matching method_details, including nils for non matched items
+    # 
+    # N.B Columns that could not be mapped are left in the array as NIL
+    # 
+    # This is to support clients that need to map via the index on @method_details
+    # 
+    # Other callers can simply call compact on the results if the index not important.
+    # 
+    # The Methoddetails instance will contain a pointer to the column index from which it was mapped.
+    # 
+    # 
+    # Options:
+    # 
+    #   [:force_inclusion]  : List of columns that do not map to any operator but should be includeed in processing.
+    #                     
+    #       This provides the opportunity for loaders to provide specific methods to handle these fields
+    #       when no direct operator is available on the modle or it's associations
+    #       
+    #   [:include_all]      : Include all headers in processing - takes precedence of :force_inclusion
     #
-    def map_inbound_to_methods( klass, columns, options = {} )
+    def map_inbound_headers_to_methods( klass, columns, options = {} )
+      
+      # If klass not in MethodDictionary yet, add to dictionary all possible operators on klass
+      # which can be used to map headers and populate an object of type klass
+      unless(MethodDictionary::for?(klass))
+        DataShift::MethodDictionary.find_operators(klass)
+        
+        DataShift::MethodDictionary.build_method_details(klass)
+      end 
       
       forced = [*options[:force_inclusion]].compact
       forced.collect! { |f| f.downcase }
       
       @method_details, @missing_methods = [], []
     
-      columns.each do |name|
-        if(name.nil? or name.empty?)
+      columns.each_with_index do |col_data, col_index|
+        
+        raw_col_data = col_data.to_s
+        
+        if(raw_col_data.nil? or raw_col_data.empty?)
           logger.warn("Column list contains empty or null columns") 
           @method_details << nil
           next
         end
         
-        operator, lookup = name.split(MethodMapper::column_delim) 
+        raw_col_name, lookup = raw_col_data.split(MethodMapper::column_delim) 
        
-        md = MethodDictionary::find_method_detail( klass, operator )
+        md = MethodDictionary::find_method_detail( klass, raw_col_name )
         
         # TODO be nice if we could cheeck that the assoc on klass responds to the specified
         # lookup key now (nice n early)
         # active_record_helper = "find_by_#{lookup}"
-        if(md.nil? && forced.include?(operator.downcase))
-          md = MethodDictionary::add(klass, operator)
+        if(md.nil? && options[:include_all] || forced.include?(raw_col_name.downcase))
+          md = MethodDictionary::add(klass, raw_col_name)
         end
         
         if(md)
           
+          md.name = raw_col_name
+          md.column_index = col_index
+          
           if(lookup)
             find_by, find_value = lookup.split(MethodMapper::column_delim) 
             md.find_by_value    = find_value 
             md.find_by_operator = find_by # TODO and klass.x.respond_to?(active_record_helper))
             #puts "DEBUG: Method Detail #{md.name};#{md.operator} : find_by_operator #{md.find_by_operator}"
           end
-              
-          @method_details << md
         else
-          @missing_methods << operator
+          # TODO populate unmapped with a real MethodDetail that is 'null' and create is_nil
+          @missing_methods << raw_col_name
         end
         
+        @method_details << md
+        
       end
-      #@method_details.compact!  .. currently we may need to map via the index on @method_details so don't remove nils for now
+     
       @method_details
     end
 
+    
+    # TODO populate unmapped with a real MethodDetail that is 'null' and create is_nil
+    # 
     # The raw client supplied names
     def method_names()
-      @method_details.collect( &:name )
+      @method_details.compact.collect( &:name )
     end
 
     # The true operator names discovered from model
     def operator_names()
-      @method_details.collect( &:operator )
+      @method_details.compact.collect( &:operator )
     end
 
+    
     # Returns true if discovered methods contain every operator in mandatory_list
     def contains_mandatory?( mandatory_list )
       [ [*mandatory_list] - operator_names].flatten.empty?