datashift 0.0.2 → 0.1.0

data/lib/loaders/spree/image_loader.rb

@@ -7,39 +7,58 @@ require 'loader_base'
 
 module DataShift
 
-  class ImageLoader < LoaderBase
-
-    def initialize(image = nil)
-      super( Image, image )
-      raise "Failed to create Image for loading" unless @load_object
-    end
+   module ImageLoading
 
+     
     # Note the Spree Image model sets default storage path to
     # => :path => ":rails_root/public/assets/products/:id/:style/:basename.:extension"
 
-    def process( image_path, record = nil)
+    def create(image_path, viewable_record = nil, options = {})
 
+      image = Image.new
+      
       unless File.exists?(image_path)
         puts "ERROR : Invalid Path"
-        return
+        return image
       end
 
-      alt = (record and record.respond_to? :name) ? record.name : ""
-
-      @load_object.alt = alt
+      alt = if(options[:alt])
+        options[:alt]
+      else
+        (viewable_record and viewable_record.respond_to? :name) ? viewable_record.name : ""
+      end
+      
+      image.alt = alt
 
       begin
-        @load_object.attachment = File.new(image_path, "r")
+        image.attachment = File.new(image_path, "r")
       rescue => e
         puts e.inspect
         puts "ERROR : Failed to read image #{image_path}"
-        return
+        return image
       end
 
-      @load_object.attachment.reprocess!
-      @load_object.viewable = record if record
+      image.attachment.reprocess!
+      image.viewable =  viewable_record if viewable_record
+
+      puts image.save ? "Success: Crteated Image: #{image.inspect}" : "ERROR : Problem saving to DB Image: #{image.inspect}"
+    end
+  end
+  
+  class ImageLoader < LoaderBase
+
+    include DataShift::ImageLoading
+        
+    def initialize(image = nil)
+      super( Image, image )
+      raise "Failed to create Image for loading" unless @load_object
+    end
 
-      puts @load_object.save ? "Success: Uploaded Image: #{@load_object.inspect}" : "ERROR : Problem saving to DB Image: #{@load_object}"
+    # Note the Spree Image model sets default storage path to
+    # => :path => ":rails_root/public/assets/products/:id/:style/:basename.:extension"
+
+    def process( image_path, record = nil)
+      @load_object = create_image(path, record)
     end
   end
 

data/lib/loaders/spree/product_loader.rb

@@ -6,8 +6,9 @@
 # Details::   Specific over-rides/additions to support Spree Products
 #
 require 'loader_base'
-require 'excel_loader'
 require 'csv_loader'
+require 'excel_loader'
+require 'image_loader'
 
 module DataShift
 
@@ -17,6 +18,7 @@ module DataShift
 
       include DataShift::CsvLoading
       include DataShift::ExcelLoading
+      include DataShift::ImageLoading
 
       def initialize(product = nil)
         super( Product, product, :instance_methods => true  )
@@ -63,6 +65,10 @@ module DataShift
         elsif(@current_method_detail.operator?('product_properties') && current_value)
 
           add_properties
+
+        elsif(@current_method_detail.operator?('images') && current_value)
+
+          add_images
           
         elsif(@current_method_detail.operator?('count_on_hand') || @current_method_detail.operator?('on_hand') )
 
@@ -150,7 +156,27 @@ module DataShift
 
       end
 
+    
+      # Special case for Images
+      # A list of paths to Images with a optional 'alt' value - supplied in form :
+      #   path:alt|path2:alt2|path_3:alt3 etc
+      #
+      def add_images
+        # TODO smart column ordering to ensure always valid by time we get to associations
+        save_if_new
+
+        images = current_value.split(LoaderBase::multi_assoc_delim)
 
+        images.each do |image|
+          
+          img_path, alt_text = image.split(LoaderBase::name_value_delim)
+          
+          image = create_image(img_path, @load_object, :alt => alt_text)
+        end
+      
+      end
+
+      
       # Special case for ProductProperties since it can have additional value applied.
       # A list of Properties with a optional Value - supplied in form :
       #   property.name:value|property.name|property.name:value

data/spec/csv_loader_spec.rb

@@ -23,9 +23,9 @@ describe 'CSV Loader' do
   end
   
   before(:each) do
-    MethodMapper.clear
-    MethodMapper.find_operators( @klazz )
-    MethodMapper.find_operators( @assoc_klazz )
+    MethodDictionary.clear
+    MethodDictionary.find_operators( @klazz )
+    MethodDictionary.find_operators( @assoc_klazz )
   end
 
 end

data/spec/excel_exporter_spec.rb

@@ -0,0 +1,79 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2011
+# License::   MIT
+#
+# Details::   Specs for Excel aspect of Active Record Loader
+#
+require File.dirname(__FILE__) + '/spec_helper'
+
+if(Guards::jruby?)
+  require 'erb'
+  require 'excel_generator'
+
+  include DataShift
+
+  describe 'Excel Generator' do
+
+    before(:all) do
+      db_connect( 'test_file' )    # , test_memory, test_mysql
+
+      # load our test model definitions - Project etc
+      require File.join($DataShiftFixturePath, 'test_model_defs')  
+   
+      # handle migration changes or reset of test DB
+      migrate_up
+
+      db_clear()    # todo read up about proper transactional fixtures
+      results_clear()
+
+      @klazz = Project
+      @assoc_klazz = Category
+    end
+  
+    before(:each) do
+      MethodDictionary.clear
+      MethodDictionary.find_operators( @klazz )
+      MethodDictionary.find_operators( @assoc_klazz )
+    end
+  
+    it "should be able to create a new excel generator" do
+      generator = ExcelGenerator.new( 'dummy.xls' )
+      
+      generator.should_not be_nil
+    end
+  
+    it "should generate template .xls file from model" do
+
+      expect = result_file('project_template_spec.xls')
+
+      gen = ExcelGenerator.new( expect )
+    
+      gen.generate(Project)
+ 
+      File.exists?(expect).should be_true
+      
+      puts "Can manually check file @ #{expect}"
+    end
+
+    it "should export a simple model to .xls spreedsheet" do
+
+      Project.create( :value_as_string	=> 'Value as Text', :value_as_boolean => true,	:value_as_double => 75.672)
+      #001 Demo string	blah blah	2011-02-14	1.00	320.00
+
+      expect= result_file('simple_export_spec.xls')
+
+      gen = ExcelGenerator.new(expect)
+
+      items = Project.all
+
+      gen.export(items)
+
+      File.exists?(expect).should be_true
+
+    end
+
+  end
+else
+  puts "WARNING: skipped excel_generator_spec : Requires JRUBY - JExcelFile requires JAVA"
+end # jruby

data/spec/excel_generator_spec.rb

@@ -32,9 +32,9 @@ if(Guards::jruby?)
     end
   
     before(:each) do
-      MethodMapper.clear
-      MethodMapper.find_operators( @klazz )
-      MethodMapper.find_operators( @assoc_klazz )
+      MethodDictionary.clear
+      MethodDictionary.find_operators( @klazz )
+      MethodDictionary.find_operators( @assoc_klazz )
     end
   
     it "should be able to create a new excel generator" do

data/spec/excel_loader_spec.rb

@@ -28,7 +28,7 @@ if(Guards::jruby?)
       db_clear()    # todo read up about proper transactional fixtures
 
 
-      @klazz = Project
+      Project = Project
       @assoc_klazz = Category
     end
   
@@ -36,49 +36,49 @@ if(Guards::jruby?)
 
       Project.delete_all
     
-      %w{category_001 category_002 category_003}.each do |cat|
+      %w{category_001 category_002 category_003 category_004 category_005}.each do |cat|
         @assoc_klazz.find_or_create_by_reference(cat)
       end
 
 
-      MethodMapper.clear
-      MethodMapper.find_operators( @klazz )
-      MethodMapper.find_operators( @assoc_klazz )
+      MethodDictionary.clear
+      MethodDictionary.find_operators( Project )
+      MethodDictionary.find_operators( @assoc_klazz )
     end
   
     it "should be able to create a new excel loader and load object" do
-      loader = ExcelLoader.new( @klazz)
+      loader = ExcelLoader.new( Project)
     
       loader.load_object.should_not be_nil
-      loader.load_object.should be_is_a(@klazz)
+      loader.load_object.should be_is_a(Project)
       loader.load_object.new_record?.should be_true
     end
   
     it "should process a simple .xls spreedsheet" do
   
-      loader = ExcelLoader.new(@klazz)
+      loader = ExcelLoader.new(Project)
  
-      count = @klazz.count
+      count = Project.count
       loader.perform_load( $DataShiftFixturePath + '/SimpleProjects.xls')
   
-      loader.loaded_count.should == (@klazz.count - count)
+      loader.loaded_count.should == (Project.count - count)
   
     end
 
     it "should process multiple associationss from single column" do
 
-      @klazz.find_by_title('001').should be_nil
-      count = @klazz.count
+      Project.find_by_title('001').should be_nil
+      count = Project.count
 
-      loader = ExcelLoader.new(@klazz)
+      loader = ExcelLoader.new(Project)
     
       loader.perform_load( $DataShiftFixturePath + '/ProjectsSingleCategories.xls')
 
       loader.loaded_count.should be > 3
-      loader.loaded_count.should == (@klazz.count - count)
+      loader.loaded_count.should == (Project.count - count)
 
       {'001' => 2, '002' => 1, '003' => 3, '099' => 0 }.each do|title, expected|
-        project = @klazz.find_by_title(title)
+        project = Project.find_by_title(title)
 
         project.should_not be_nil
         puts "#{project.inspect} [#{project.categories.size}]"
@@ -97,7 +97,7 @@ if(Guards::jruby?)
       loader.loaded_count.should == (Project.count - count)
   
       {'004' => 3, '005' => 1, '006' => 0, '007' => 1 }.each do|title, expected|
-        project = @klazz.find_by_title(title)
+        project = Project.find_by_title(title)
   
         project.should_not be_nil
 
@@ -106,6 +106,25 @@ if(Guards::jruby?)
   
     end
   
+    it "should process multiple associations with lookup specified in column from excel spreedsheet", :fail => true do
+  
+      loader = ExcelLoader.new(Project)
+  
+      count = Project.count
+      loader.perform_load( $DataShiftFixturePath + '/ProjectsMultiCategoriesHeaderLookup.xls')
+  
+      loader.loaded_count.should == (Project.count - count)
+  
+      {'004' => 4, '005' => 1, '006' => 0, '007' => 1 }.each do|title, expected|
+        project = Project.find_by_title(title)
+  
+        project.should_not be_nil
+
+        project.should have(expected).categories
+      end
+  
+    end
+    
     it "should process excel spreedsheet with extra undefined columns" do
       loader = ExcelLoader.new(Project)
       lambda {loader.perform_load( ifixture_file('BadAssociationName.xls') ) }.should_not raise_error

data/spec/loader_spec.rb

@@ -15,15 +15,15 @@ describe 'Basic Loader' do
     db_connect( 'test_file' )    # , test_memory, test_mysql
     
     # load our test model definitions - Project etc
-    require File.join($DataShiftFixturePath, 'test_model_defs')  
-   
+    require File.join($DataShiftFixturePath, 'test_model_defs') 
+      
     migrate_up
     @klazz = Project
   end
   
   before(:each) do
-    MethodMapper.clear
-    MethodMapper.find_operators( @klazz )
+    MethodDictionary.clear
+    MethodDictionary.find_operators( @klazz )
   end
   
   it "should be able to create a new loader and load object" do

data/spec/method_dictionary_spec.rb

@@ -0,0 +1,243 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2011
+# License::   MIT
+#
+# Details::   Specs for MethodMapper aspect of Active Record Loader
+#             MethodMapper provides the bridge between 'strings' e.g column headings
+#             and a classes different types of assignment operators
+#
+require File.dirname(__FILE__) + '/spec_helper'
+    
+describe 'Method Mapping' do
+
+  before(:all) do
+    db_connect( 'test_file' )    # , test_memory, test_mysql
+    
+    # load our test model definitions - Project etc
+    require ifixture_file('test_model_defs')  
+
+    migrate_up
+  end
+  
+  before(:each) do
+    MethodDictionary.clear
+    MethodDictionary.find_operators( Project )
+    MethodDictionary.find_operators( Milestone )
+  end
+  
+  it "should store dictionary for multiple AR models" do
+    MethodDictionary.assignments.size.should == 2 
+    MethodDictionary.has_many.size.should == 2
+  end
+  
+  it "should populate method dictionary for a given AR model" do
+
+    MethodDictionary.has_many.should_not be_empty
+    MethodDictionary.has_many[Project].should include('milestones')
+
+    MethodDictionary.assignments.should_not be_empty
+    MethodDictionary.assignments[Project].should include('id')
+    MethodDictionary.assignments[Project].should include('value_as_string')
+    MethodDictionary.assignments[Project].should include('value_as_text')
+
+    MethodDictionary.belongs_to.should_not be_empty
+    MethodDictionary.belongs_to[Project].should be_empty
+
+
+    MethodDictionary.column_types.should be_is_a(Hash)
+    MethodDictionary.column_types.should_not be_empty
+    MethodDictionary.column_types[Project].size.should == Project.columns.size
+
+
+  end
+
+  it "should populate assigment members without the equivalent association names" do
+
+    # we should remove has-many & belongs_to from basic assignment set as they require a DB lookup
+    # or a Model.create call, not a simple assignment
+
+    MethodDictionary.assignments_for(Project).should_not include( MethodDictionary.belongs_to_for(Project) )
+    MethodDictionary.assignments_for(Project).should_not include( MethodDictionary.has_many_for(Project) )
+  end
+
+
+  it "should populate assignment operators for method details for different forms of a column name" do
+
+    MethodDictionary.build_method_details( Project )
+    
+    [:value_as_string, 'value_as_string', "VALUE as_STRING", "value as string"].each do |format|
+
+      method_details = MethodDictionary.find_method_detail( Project, format )
+
+      method_details.class.should == MethodDetail
+
+      method_details.operator.should == 'value_as_string'
+      method_details.operator_for(:assignment).should == 'value_as_string'
+      
+      method_details.operator?('value_as_string').should be_true
+      method_details.operator?('blah_as_string').should be_false
+
+      method_details.operator_for(:belongs_to).should be_nil
+      method_details.operator_for(:has_many).should be_nil
+    end
+  end
+
+
+  # Note : Not all assignments will currently have a column type, for example
+  # those that are derived from a delegate_belongs_to
+
+  it "should populate column types for assignment operators in method details" do
+
+    MethodDictionary.build_method_details( Project )
+        
+    [:value_as_string, 'value_as_string', "VALUE as_STRING", "value as string"].each do |format|
+
+      method_details = MethodDictionary.find_method_detail( Project, format )
+
+      method_details.class.should == MethodDetail
+
+      method_details.col_type.should_not be_nil
+      method_details.col_type.name.should == 'value_as_string'
+      method_details.col_type.default.should == nil
+      method_details.col_type.sql_type.should include 'varchar(255)'   # db specific, sqlite
+      method_details.col_type.type.should == :string
+    end
+  end
+
+  it "should populate required Class for assignment operators based on column type" do
+
+    MethodDictionary.build_method_details( Project )
+        
+    [:value_as_string, 'value_as_string', "VALUE as_STRING", "value as string"].each do |format|
+
+      method_details = MethodDictionary.find_method_detail( Project, format )
+
+      method_details.operator_class_name.should == 'String'
+      method_details.operator_class.should be_is_a(Class)
+      method_details.operator_class.should == String
+    end
+
+  end
+
+  it "should populate belongs_to operator for method details for different forms of a column name" do
+
+    MethodDictionary.build_method_details( Project )
+    MethodDictionary.build_method_details( Milestone )
+            
+    # milestone.project = project.id
+    [:project, 'project', "PROJECT", "prOJECt"].each do |format|
+
+      method_details = MethodDictionary.find_method_detail( Milestone, format )
+
+      method_details.should_not be_nil
+
+      method_details.operator.should == 'project'
+      method_details.operator_for(:belongs_to).should == 'project'
+
+      method_details.operator_for(:assignment).should be_nil
+      method_details.operator_for(:has_many).should be_nil
+    end
+
+  end
+
+  it "should populate required Class for belongs_to operator method details" do
+
+    MethodDictionary.find_operators( LoaderRelease )
+    MethodDictionary.find_operators( LongAndComplexTableLinkedToVersion )
+
+    MethodDictionary.build_method_details( LoaderRelease )
+    MethodDictionary.build_method_details( LongAndComplexTableLinkedToVersion )
+          
+    
+    # release.project = project.id
+    [:project, 'project', "PROJECT", "prOJECt"].each do |format|
+
+      method_details = MethodDictionary.find_method_detail( LoaderRelease, format )
+
+      method_details.operator_class_name.should == 'Project'
+      method_details.operator_class.should == Project
+    end
+
+
+    [:version, "Version", "verSION"].each do |format|
+      method_details = MethodDictionary.find_method_detail( LongAndComplexTableLinkedToVersion, format )
+
+      method_details.operator_type.should == :belongs_to
+
+      method_details.operator_class_name.should == 'Version'
+      method_details.operator_class.should == Version
+    end
+  end
+
+   it "should populate required Class for has_one operator method details" do
+
+    MethodDictionary.find_operators( Version )
+    MethodDictionary.build_method_details( Version )
+       
+    # version.long_and_complex_table_linked_to_version = LongAndComplexTableLinkedToVersion.create()
+
+    [:long_and_complex_table_linked_to_version, 'LongAndComplexTableLinkedToVersion', "Long And Complex_Table_Linked To  Version", "Long_And_Complex_Table_Linked_To_Version"].each do |format|
+      method_details = MethodDictionary.find_method_detail( Version, format )
+
+      method_details.should_not be_nil
+
+      method_details.operator.should == 'long_and_complex_table_linked_to_version'
+      
+      method_details.operator_type.should == :has_one
+
+      method_details.operator_class_name.should == 'LongAndComplexTableLinkedToVersion'
+      method_details.operator_class.should == LongAndComplexTableLinkedToVersion
+    end
+  end
+
+
+  it "should find has_many operator for method details" do
+
+    MethodDictionary.build_method_details( Project )
+     
+    [:milestones, "Mile Stones", 'mileSTONES', 'MileStones'].each do |format|
+
+      method_details = MethodDictionary.find_method_detail( Project, format )
+
+      method_details.class.should == MethodDetail
+      
+      result = 'milestones'
+      method_details.operator.should == result
+      method_details.operator_for(:has_many).should == result
+
+      method_details.operator_for(:belongs_to).should be_nil
+      method_details.operator_for(:assignments).should be_nil
+    end
+
+  end
+
+ 
+  it "should return nil when non existent column name" do
+    ["On sale", 'on_sale'].each do |format|
+      detail = MethodDictionary.find_method_detail( Project, format )
+
+      detail.should be_nil
+    end
+  end
+
+
+  it "should find a set of methods based on a list of column names" do
+     pending("key API - map column headers to set of methods")
+  end
+
+  it "should not by default map setter methods", :fail => true do
+    MethodDictionary.assignments[Milestone].should_not include('title')
+  end
+  
+  it "should support reload and  inclusion of setter methods", :fail => true do
+
+    MethodDictionary.assignments[Milestone].should_not include('title')
+        
+    MethodDictionary.find_operators( Milestone, :reload => true, :instance_methods => true )
+     
+    # Milestone delegates :title to Project
+    MethodDictionary.assignments[Milestone].should include('title')
+  end
+
+end