datashift 0.0.2 → 0.1.0

data/lib/datashift/method_mapper.rb

@@ -16,6 +16,7 @@
 #             This real association can then be used to send spreadsheet row data to the AR object.
 #             
 require 'method_detail'
+require 'method_dictionary'
 
 module DataShift
 
@@ -24,11 +25,19 @@ module DataShift
     attr_accessor :header_row, :headers
     attr_accessor :method_details, :missing_methods
   
-    @@has_many     = Hash.new
-    @@belongs_to   = Hash.new
-    @@assignments  = Hash.new
-    @@column_types = Hash.new
+    
+    # As well as just the column name, support embedding find operators for that column
+    # in the heading .. i.e Column header => 'BlogPosts:user_id' 
+    # ... association has many BlogPosts selected via find_by_user_id
+    # 
+    def self.column_delim
+      @column_delim ||= ':'
+      @column_delim
+    end
 
+    def self.set_column_delim(x)  @column_delim = x; end
+    
+  
     def initialize
       @method_details = []
       @headers = []
@@ -39,14 +48,22 @@ module DataShift
     # not be exactly as required e.g handles capitalisation, white space, _ etc
     # Returns: Array of matching method_details
     #
-    def populate_methods( klass, method_list )
+    def map_inbound_to_methods( klass, method_list )
+      
       @method_details, @missing_methods = [], []
     
-      method_list.each do |x|
-        md = MethodMapper::find_method_detail( klass, x )
+      method_list.each do |name|
+        x, lookup = name.split(MethodMapper::column_delim) 
+        md = MethodDictionary::find_method_detail( klass, x )
+        
+        # TODO be nice if we could cheeck that the assoc on klass responds to the specified
+        # lookup key now (nice n early)
+        # active_record_helper = "find_by_#{lookup}"
+        
+        md.find_by_operator = lookup if(lookup) # TODO and klass.x.respond_to?(active_record_helper))
         md ? @method_details << md : @missing_methods << x
       end
-      #@method_details.compact!  .. currently we may neeed to map via the index on @method_details so don't remove nils for now
+      #@method_details.compact!  .. currently we may need to map via the index on @method_details so don't remove nils for now
       @method_details
     end
 
@@ -69,189 +86,6 @@ module DataShift
       [ [*mandatory_list] - operator_names].flatten
     end
 
-    # Create picture of the operators for assignment available on an AR model,
-    # including via associations (which provide both << and = )
-    # Options:
-    # :reload => clear caches and reperform  lookup
-    # :instance_methods => if true include instance method type assignment operators as well as model's pure columns
-    #
-    def self.find_operators(klass, options = {} )
-
-      # Find the has_many associations which can be populated via <<
-      if( options[:reload] || @@has_many[klass].nil? )
-        @@has_many[klass] = klass.reflect_on_all_associations(:has_many).map { |i| i.name.to_s }
-        klass.reflect_on_all_associations(:has_and_belongs_to_many).inject(@@has_many[klass]) { |x,i| x << i.name.to_s }
-      end
-      # puts "DEBUG: Has Many Associations:", @@has_many[klass].inspect
-
-      # Find the belongs_to associations which can be populated via  Model.belongs_to_name = OtherArModelObject
-      if( options[:reload] || @@belongs_to[klass].nil? )
-        @@belongs_to[klass] = klass.reflect_on_all_associations(:belongs_to).map { |i| i.name.to_s }
-      end
-
-      #puts "Belongs To Associations:", @@belongs_to[klass].inspect
-
-      # Find the has_one associations which can be populated via  Model.has_one_name = OtherArModelObject
-      if( options[:reload] || self.has_one[klass].nil? )
-        self.has_one[klass] = klass.reflect_on_all_associations(:has_one).map { |i| i.name.to_s }
-      end
-
-      #puts "has_one Associations:", self.has_one[klass].inspect
-
-      # Find the model's column associations which can be populated via xxxxxx= value
-      # Note, not all reflections return method names in same style so we convert all to
-      # the raw form i.e without the '='  for consistency 
-      if( options[:reload] || @@assignments[klass].nil? )
-
-        @@assignments[klass] = klass.column_names
-           
-        if(options[:instance_methods] == true)
-          setters = klass.instance_methods.grep(/\w+=/).collect {|x| x.to_s }
-
-          if(klass.respond_to? :defined_activerecord_methods)
-            setters = setters - klass.defined_activerecord_methods.to_a
-          end
-
-          # get into same format as other names 
-          @@assignments[klass] += setters.map{|i| i.gsub(/=/, '')}
-        end
-        
-        @@assignments[klass] -= @@has_many[klass] if(@@has_many[klass])
-        @@assignments[klass] -= @@belongs_to[klass] if(@@belongs_to[klass])
-        @@assignments[klass] -= self.has_one[klass] if(self.has_one[klass])
- 
-        @@assignments[klass].uniq!
-
-        @@assignments[klass].each do |assign|
-          @@column_types[klass] ||= {}
-          column_def = klass.columns.find{ |col| col.name == assign }
-          @@column_types[klass].merge!( assign => column_def) if column_def
-        end
-      end
-    end
-
-    def self.build_method_details( klass )
-      @method_details ||= {}
-      
-      @method_details[klass] = []
-      
-      assignments_for(klass).each do |n|
-        @method_details[klass] << MethodDetail.new(n, klass, n, :assignment)
-      end
-        
-      has_one_for(klass).each do |n|
-        @method_details[klass] << MethodDetail.new(n, klass, n, :has_one)
-      end
-        
-      has_many_for(klass).each do |n|
-        @method_details[klass] << MethodDetail.new(n, klass, n, :has_many)
-      end
-        
-      belongs_to_for(klass).each do |n|
-        @method_details[klass] << MethodDetail.new(n, klass, n, :belongs_to)
-      end
-    end
-    
-    def self.method_details
-      @method_details ||= {}
-      @method_details
-    end
-    
-    # Find the proper format of name, appropriate call + column type for a given name.
-    # e.g Given users entry in spread sheet check for pluralization, missing underscores etc
-    #
-    # If not nil, returned method can be used directly in for example klass.new.send( call, .... )
-    #
-    def self.find_method_detail( klass, external_name )
-      operator = nil
-
-      name = external_name.to_s
-
-      # TODO - check out regexp to do this work better plus Inflections ??
-      # Want to be able to handle any of ["Count On hand", 'count_on_hand', "Count OnHand", "COUNT ONHand" etc]
-      [
-        name,
-        name.tableize,
-        name.gsub(' ', '_'),
-        name.gsub(' ', '_').downcase,
-        name.gsub(/(\s+)/, '_').downcase,
-        name.gsub(' ', ''),
-        name.gsub(' ', '').downcase,
-        name.gsub(' ', '_').underscore].each do |n|
-      
-        operator = (assignments_for(klass).include?(n)) ? n : nil
-      
-        return MethodDetail.new(name, klass, operator, :assignment, @@column_types[klass]) if(operator)
-
-        operator = (has_one_for(klass).include?(n)) ? n : nil
-      
-        return MethodDetail.new(name, klass, operator, :has_one, @@column_types[klass]) if(operator)
-
-        operator = (has_many_for(klass).include?(n)) ?  n : nil
-      
-        return MethodDetail.new(name, klass, operator, :has_many, @@column_types[klass]) if(operator)
-      
-        operator = (belongs_to_for(klass).include?(n)) ? n : nil
-      
-        return MethodDetail.new(name, klass, operator, :belongs_to, @@column_types[klass]) if(operator)
-      
-      end
-
-      nil
-    end
-
-    def self.clear
-      @@belongs_to.clear
-      @@has_many.clear
-      @@assignments.clear
-      @@column_types.clear
-      self.has_one.clear
-    end
-
-    def self.column_key(klass, column)
-      "#{klass.name}:#{column}"
-    end
-
-    # TODO - remove use of class variables - not good Ruby design
-    def self.belongs_to
-      @@belongs_to
-    end
-
-    def self.has_many
-      @@has_many
-    end
-
-    def self.has_one
-      @has_one ||= {}
-      @has_one
-    end
-
-    def self.assignments
-      @@assignments
-    end
-    def self.column_types
-      @@column_types
-    end
-
-
-    def self.belongs_to_for(klass)
-      @@belongs_to[klass] || []
-    end
-    def self.has_many_for(klass)
-      @@has_many[klass] || []
-    end
-
-    def self.has_one_for(klass)
-      self.has_one[klass] || []
-    end
-
-    def self.assignments_for(klass)
-      @@assignments[klass] || []
-    end
-    def self.column_type_for(klass, column)
-      @@column_types[klass] ?  @@column_types[klass][column] : []
-    end
-  
   end
 
 end

data/lib/generators/excel_generator.rb

@@ -10,10 +10,11 @@
 #
 module DataShift
 
+  require 'generator_base'
+      
   if(Guards::jruby?)
 
     require 'jruby/jexcel_file'
-    require 'generator_base'
 
     class ExcelGenerator < GeneratorBase
 
@@ -26,7 +27,7 @@ module DataShift
       # Create an Excel file template (header row) representing supplied Model
     
       def generate(model, options = {})
-        MethodMapper.find_operators( model )
+        MethodDictionary.find_operators( model )
 
         @filename = options[:filename] if  options[:filename]
 
@@ -40,7 +41,7 @@ module DataShift
         
         raise "Failed to create Excel WorkSheet for #{model.name}" unless excel.sheet
 
-        excel.set_headers(MethodMapper.assignments[model])
+        excel.set_headers(MethodDictionary.assignments[model])
 
         excel.save( @filename )
       end
@@ -78,20 +79,20 @@ module DataShift
           excel.create_sheet( items.first.class.name )
         end
         
-        MethodMapper.find_operators( klass )
+        MethodDictionary.find_operators( klass )
          
-        MethodMapper.build_method_details( klass )
+        MethodDictionary.build_method_details( klass )
           
-        work_list = (options[:with]) ? options[:with] : [:assignment, :belongs_to, :has_one, :has_many]
+        work_list = (options[:with]) ? options[:with] : [:assignments, :belongs_to, :has_one, :has_many]
         
         headers = []
         
-        MethodMapper.method_details[klass].each do |method_detail|   
-          if(method_detail.operator_type == :assignment)
-            headers << "#{klass}:#{method_detail.operator}"
-          end
+        work_list.each do |mdtype|
+          method_details = MethodDictionary.send("#{mdtype}_for", klass)
+        
+          method_details.each {|md| headers << "#{md.operator}" }
         end
-      
+        
         excel.set_headers( headers )
                 
         data = []

data/lib/helpers/spree_helper.rb

@@ -5,6 +5,8 @@
 #
 # Details::   Spree Helper mixing in Support for testing or loading Rails Spree e-commerce.
 # 
+#             The Spree version you want to test should be picked up from the Gemfile
+# 
 #             Since datashift gem is not a Rails app or a Spree App, provides utilities to internally
 #             create a Spree Database, and to load Spree components, enabling standalone testing.
 #
@@ -41,20 +43,39 @@ module Spree
   end
     
   def self.boot
-    
+ 
     require 'spree'
     require 'spree_core'
 
-    $LOAD_PATH << root << lib_root << app_root << File.join(app_root, 'models')
+    #require 'rake'
+    #require 'rubygems/package_task'
+    #require 'thor/group'
+    require File.expand_path( lib_root + '/generators/spree/install/install_generator')
+    require 'spree/core/testing_support/common_rake'
 
-    require 'spree_core/preferences/model_hooks'
     
-    # Initialize preference system
-    ActiveRecord::Base.class_eval do
-      include Spree::Preferences
-      include Spree::Preferences::ModelHooks
-    end
+    Spree::SandboxGenerator.start ["--lib_name=spree", "--database=#{ENV['DB_NAME']}"]
+    Spree::InstallGenerator.start ["--auto-accept"]
 
+    cmd = "bundle exec rake assets:precompile:nondigest"; 
+    puts cmd; system cmd
+
+    
+    return
+    
+ 
+    # TODO how to check gem version actually loaded and do conditional
+    #
+    #if(PRE 1)
+      #require 'spree_core/preferences/model_hooks'
+    #
+    ## Initialize preference system
+    # ActiveRecord::Base.class_eval do
+    #   include Spree::Preferences
+    #   include Spree::Preferences::ModelHooks
+    # end
+    #end
+    
     gem 'paperclip'
     gem 'nested_set'
 
@@ -71,7 +92,8 @@ module Spree
 
     ActiveRecord::Base.send(:include, ActiveMerchant::Billing)
 
-    require 'scopes'
+     
+    #require 'scopes'
     
     # Not sure how Rails manages this seems lots of circular dependencies so
     # keep trying stuff till no more errors
@@ -106,14 +128,16 @@ module Spree
       end
     end
 
-    require 'product'
-    require 'lib/product_filters'
+    #if(PRE 1)require 'product'
+    #require 'lib/product_filters'
+    
+    
     load_models( true )
 
   end
 
   def self.load_models( report_errors = nil )
-    puts 'load from', root
+    puts 'Loading Spree models from', root
     Dir[root + '/app/models/**/*.rb'].each {|r|
       begin
         require r if File.file?(r)

data/lib/loaders/excel_loader.rb

@@ -51,7 +51,7 @@ module DataShift
         raise MissingHeadersError, "No headers found - Check Sheet #{@sheet} is complete and Row #{header_row_index} contains headers" unless(@header_row)
 
         @headers = []
-        
+        category_003
         (0..JExcelFile::MAX_COLUMNS).each do |i|
           cell = @header_row.getCell(i)
           break unless cell
@@ -63,6 +63,7 @@ module DataShift
         raise MissingHeadersError, "No headers found - Check Sheet #{@sheet} is complete and Row #{header_row_index} contains headers" if(@headers.empty?)
 
         # Create a method_mapper which maps list of headers into suitable calls on the Active Record class
+        # For example if model has an attribute 'price' will map columns called Price, price, PRICE etc to this attribute
         map_headers_to_operators( @headers, options[:strict] , @mandatory )
 
         logger.info "Excel Loader prcoessing #{@excel.num_rows} rows"

data/lib/loaders/loader_base.rb

@@ -81,18 +81,23 @@ module DataShift
       @multi_assoc_delim
     end
 
-    
+
     def self.set_multi_assoc_delim(x) @multi_assoc_delim = x; end
 
+    
     # Options
     #     :instance_methods => true
 
     def initialize(object_class, object = nil, options = {})
       @load_object_class = object_class
 
-      # Gather list of all possible 'setter' methods on AR class (instance variables and associations)
-      DataShift::MethodMapper.find_operators( @load_object_class, :reload => true, :instance_methods => options[:instance_methods] )
+      # Gather names of all possible 'setter' methods on AR class (instance variables and associations)
+      DataShift::MethodDictionary.find_operators( @load_object_class, :reload => true, :instance_methods => options[:instance_methods] )
 
+      # Create dictionary of data on all possible 'setter' methods which can be used to
+      # populate or integrate an object of type @load_object_class
+      DataShift::MethodDictionary.build_method_details(@load_object_class)
+      
       @method_mapper = DataShift::MethodMapper.new
       @options = options.clone
       @headers = []
@@ -115,15 +120,18 @@ module DataShift
     end
 
     
-    # Core API - Given a list of free text column names from a file, map all headers to
-    # method mapper's operator list.
+    # Core API - Given a list of free text column names from a file, 
+    # map all headers to a method detail containing operator details.
+    # 
+    # This is then available through @method_mapper.method_details.each
+    # 
     # Options:
     #  strict : report any header values that can't be mapped as an error
     #
     def map_headers_to_operators( headers, strict, mandatory = [])
       @headers = headers
       
-      @method_mapper.populate_methods( load_object_class, @headers )
+      method_details = @method_mapper.map_inbound_to_methods( load_object_class, @headers )
 
       unless(@method_mapper.missing_methods.empty?)
         puts "WARNING: Following column headings could not be mapped : #{@method_mapper.missing_methods.inspect}"
@@ -137,12 +145,12 @@ module DataShift
     end
 
 
-    # Core API - Given a free text column name from a file, search method mapper for
+    # Core API - Given a single free text column name from a file, search method mapper for
     # associated operator on base object class.
     # 
     # If suitable association found, process row data and then assign to current load_object
     def find_and_process(column_name, data)
-      method_detail = MethodMapper.find_method_detail( load_object_class, column_name )
+      method_detail = MethodDictionary.find_method_detail( load_object_class, column_name )
 
       if(method_detail)
         prepare_data(method_detail, data)
@@ -242,33 +250,42 @@ module DataShift
           save_if_new
 
           # A single column can contain multiple associations delimited by special char
+          # Size:large|Colour:red,green,blue => ['Size:large', 'Colour:red,green,blue']
           columns = @current_value.to_s.split( LoaderBase::multi_assoc_delim)
 
           # Size:large|Colour:red,green,blue   => generates find_by_size( 'large' ) and find_all_by_colour( ['red','green','blue'] )
 
-          columns.each do |assoc|
-            operator, values = assoc.split(LoaderBase::name_value_delim)
-
-            lookups = values.split(LoaderBase::multi_value_delim)
+          columns.each do |col_str|
+            
+            find_operator, col_values = "",""
+           
+            if(@current_method_detail.find_by_operator)
+              find_operator, col_values = @current_method_detail.find_by_operator, col_str
+            else
+              find_operator, col_values = col_str.split(LoaderBase::name_value_delim)
+              raise "No key to find #{@current_method_detail.operator} in DB. Expected format key:value" unless(col_values)
+            end
+            
+            find_by_values = col_values.split(LoaderBase::multi_value_delim)
 
-            if(lookups.size > 1)
+            if(find_by_values.size > 1)
 
-              @current_value = @current_method_detail.operator_class.send("find_all_by_#{operator}", lookups )
+              @current_value = @current_method_detail.operator_class.send("find_all_by_#{find_operator}", find_by_values )
 
-              unless(lookups.size == @current_value.size)
-                found = @current_value.collect {|f| f.send(operator) }
-                @load_object.errors.add( method_detail.operator, "Association with key(s) #{(lookups - found).inspect} NOT found")
+              unless(find_by_values.size == @current_value.size)
+                found = @current_value.collect {|f| f.send(find_operator) }
+                @load_object.errors.add( @current_method_detail.operator, "Association with key(s) #{(find_by_values - found).inspect} NOT found")
                 puts "WARNING: Association with key(s) #{(lookups - found).inspect} NOT found - Not added."
                 next if(@current_value.empty?)
               end
 
             else
 
-              @current_value = @current_method_detail.operator_class.send("find_by_#{operator}", lookups )
+              @current_value = @current_method_detail.operator_class.send("find_by_#{find_operator}", find_by_values )
 
               unless(@current_value)
-                @load_object.errors.add( @current_method_detail.operator, "Association with key #{lookups} NOT found")
-                puts "WARNING: Association with key #{lookups} NOT found - Not added."
+                @load_object.errors.add( @current_method_detail.operator, "Association with key #{find_by_values} NOT found")
+                puts "WARNING: Association with key #{find_by_values} NOT found - Not added."
                 next
               end