datashift 0.15.0 → 0.16.0

data/spec/csv_exporter_spec.rb

@@ -13,127 +13,132 @@ require 'csv_exporter'
 describe 'CSV Exporter' do
 
   before(:all) do
-      
-    # load our test model definitions - Project etc  
-    require ifixture_file('test_model_defs')  
-  
-    db_connect( 'test_file' )    # , test_memory, test_mysql
-   
-    # handle migration changes or reset of test DB
-    migrate_up
-
-    results_clear()
+    results_clear( "*.csv" )
   end
-  
+
   before(:each) do
     DataShift::MethodDictionary.clear
     DataShift::MethodDictionary.find_operators( Project )
-    
+
     db_clear()    # todo read up about proper transactional fixtures
-    
-    Project.create( :value_as_string	=> 'Value as String', :value_as_boolean => true,	:value_as_double => 75.672)
-    Project.create( :value_as_string	=> 'Another Value as String', :value_as_boolean => false,	:value_as_double => 12)
-     
-  end
-  
-  it "should be able to create a new CSV exporter" do
-    exporter = DataShift::CsvExporter.new( 'rspec_csv_empty.csv' )
-      
-    exporter.should_not be_nil
   end
 
-  it "should throw if not active record objects" do
-    exporter = DataShift::CsvExporter.new( 'rspec_csv_empty.csv' )
-      
-    expect{ exporter.export([123.45]) }.to raise_error(ArgumentError)
-  end
-  
-
-  it "should export collection of model objects to .xls file" do
-
-    expect = result_file('project_export_spec.csv')
-
-    exporter = DataShift::CsvExporter.new( expect )
-     
-    count = Project.count 
-    
-    Project.create( :value_as_string	=> 'Value as String', :value_as_boolean => true,	:value_as_double => 75.672)
-     
-    Project.count.should == count + 1
-    
-    exporter.export(Project.all)
- 
-    File.exists?(expect).should be_true
-      
-    puts "Can manually check file @ #{expect}"
-    
-    File.foreach(expect) {}
-    count = $.
-    count.should == Project.count + 1
-  end
+  context 'simple project' do
 
-  it "should handle bad params to export" do
+    before(:each) do
+      create( :project )
+    end
 
-    expect = result_file('project_first_export_spec.csv')
+    it "should be able to create a new CSV exporter" do
+      exporter = DataShift::CsvExporter.new( 'exp_rspec_csv_empty.csv' )
 
-    exporter = DataShift::CsvExporter.new( expect )
-    
-    expect{ exporter.export(nil) }.not_to raise_error
+      expect(exporter).not_to be_nil
+    end
 
-    expect{ exporter.export([]) }.not_to raise_error
-   
-    puts "Can manually check file @ #{expect}"
-  end
-  
-  it "should export a model object to csv file" do
-
-    expect = result_file('project_first_export_spec.csv')
-
-    exporter = DataShift::CsvExporter.new( expect )
-    
-    exporter.export(Project.all[0])
- 
-    File.exists?(expect).should be_true
-      
-    puts "Can manually check file @ #{expect}"
-  end
+    it "should throw if not active record objects" do
+      exporter = DataShift::CsvExporter.new( 'exp_rspec_csv_empty.csv' )
 
-  it "should export a model and result of method calls on it to csv file" do
+      expect{ exporter.export([123.45]) }.to raise_error(ArgumentError)
+    end
 
-    expect = result_file('project_with_methods_export_spec.csv')
 
-    exporter = DataShift::CsvExporter.new( expect )
-     
-    exporter.export(Project.all, {:methods => [:multiply]})
- 
-    File.exists?(expect).should be_true
-      
-    puts "Can manually check file @ #{expect}"
-    
-    File.foreach(expect) {}
-    count = $.
-    count.should == Project.count + 1
+    it "should export collection of model objects to .xls file" do
+
+      expected = result_file('exp_project_collection_spec.csv')
+
+      exporter = DataShift::CsvExporter.new( expected )
+
+      count = Project.count
+
+      Project.create( :value_as_string	=> 'Value as String', :value_as_boolean => true,	:value_as_double => 75.672)
+
+      Project.count.should == count + 1
+
+      exporter.export(Project.all)
+
+      expect(File.exists?(expected)).to eq true
+
+      puts "Can manually check file @ #{expected}"
+
+      File.foreach(expected) {}
+      count = $.
+      count.should == Project.count + 1
+    end
+
+    it "should handle bad params to export" do
+
+      expected = result_file('project_first_export_spec.csv')
+
+      exporter = DataShift::CsvExporter.new( expected )
+
+      expect{ exporter.export(nil) }.not_to raise_error
+
+      expect{ exporter.export([]) }.not_to raise_error
+
+      puts "Can manually check file @ #{expected}"
+    end
+
+    it "should export a model object to csv file" do
+
+      expected = result_file('project_first_export_spec.csv')
+
+      exporter = DataShift::CsvExporter.new( expected )
+
+      exporter.export(Project.all[0])
+
+      expect(File.exists?(expected)).to eq true
+
+      puts "Can manually check file @ #{expected}"
+    end
+
+    it "should export a model and result of method calls on it to csv file" do
+
+      expected = result_file('project_with_methods_export_spec.csv')
+
+      exporter = DataShift::CsvExporter.new( expected )
+
+      exporter.export(Project.all, {:methods => [:multiply]})
+
+      expect(File.exists?(expected)).to eq true
+
+      puts "Can manually check file @ #{expected}"
+
+      File.foreach(expected) {}
+      count = $.
+      count.should == Project.count + 1
+    end
+
   end
-  
-  it "should export a  model and associations to .xls file" do
 
-    p = Project.create( :value_as_string	=> 'Value as String', :value_as_boolean => true,	:value_as_double => 75.672)
+  it "should export a model and associations to csv" do
 
-    p.milestones.create( :name => 'milestone_1', :cost => 23.45)
-    
-    expect= result_file('project_plus_assoc_export_spec.csv')
+    create( :project_user )
+    create_list(:project, 7)
 
-    gen = DataShift::CsvExporter.new(expect)
+    expected = result_file('exp_project_plus_assoc_export_spec.csv')
 
-    gen.export_with_associations(Project, Project.all)
+    gen = DataShift::CsvExporter.new(expected)
 
-    File.exists?(expect).should be_true
+    items = Project.all
 
-    File.foreach(expect) {}
+    gen.export_with_associations(Project, items)
+
+    File.foreach(expected) {}
     count = $.
-    count.should == Project.count + 1
-    
-  end
+    count.should == items.size + 1
+
+    expect(File.exists?(expected)).to eq true
 
+    csv = CSV.read(expected)
+
+    expect(csv[0]).to include 'owner'
+    expect(csv[0]).to include 'user'
+
+    user_inx = csv[0].index 'user'
+
+    expect(user_inx).to be > -1
+
+    expect( csv[1][user_inx] ).to include 'mr'
+  end
 
 end

data/spec/csv_generator_spec.rb

@@ -0,0 +1,159 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2011
+# License::   MIT
+#
+# Details::   Specs for Excel aspect of Active Record Loader
+#
+require File.dirname(__FILE__) + '/spec_helper'
+
+require 'erb'
+require 'csv_generator'
+
+include DataShift
+
+describe 'CSV Generator' do
+
+  before(:all) do
+    results_clear("*.csv")
+
+    @klazz = Project
+    @assoc_klazz = Category
+  end
+
+  before(:each) do
+    MethodDictionary.clear
+    MethodDictionary.find_operators( @klazz )
+    MethodDictionary.find_operators( @assoc_klazz )
+  end
+
+  it "should be able to create a new csv generator" do
+    generator = CsvGenerator.new( 'dummy.csv' )
+
+    generator.should_not be_nil
+  end
+
+  it "should generate template .csv file from model" do
+
+    expected = result_file('project_template_spec.csv')
+
+    gen = CsvGenerator.new( expected )
+
+    gen.generate(Project)
+
+    expect(File.exists?(expected)).to eq true
+
+    puts "Can manually check file @ #{expected}"
+
+    csv = CSV.read(expected)
+
+    headers = csv[0]
+
+    [ "title", "value_as_string", "value_as_text", "value_as_boolean", "value_as_datetime", "value_as_integer", "value_as_double"].each do |check|
+      headers.include?(check).should == true
+    end
+  end
+
+  # has_one  :owner
+  # has_many :milestones
+  # has_many :loader_releases
+  # has_many :versions, :through => :loader_releases
+  # has_and_belongs_to_many :categories
+
+  it "should include all associations in template .csv file from model" do
+
+    expected = result_file('project_plus_assoc_template_spec.csv')
+
+    gen = CsvGenerator.new(expected)
+
+    gen.generate_with_associations(Project)
+
+    expect( File.exists?(expected)).to eq true
+
+    csv = CSV.read(expected)
+
+    headers = csv[0]
+
+    ["owner", "milestones", "loader_releases", "versions", "categories"].each do |check|
+      headers.include?(check).should == true
+    end
+  end
+
+
+  it "should enable us to exclude associations by type in template .csv file" do
+
+    expected = result_file('project_plus_some_assoc_template_spec.csv')
+
+    gen = CsvGenerator.new(expected)
+
+    options = {:exclude => :has_many }
+
+    gen.generate_with_associations(Project, options)
+
+    expect(File.exists?(expected)).to eq true #, "Failed to find expected result file #{expected}"
+
+    csv = CSV.read(expected)
+
+    headers = csv[0]
+
+    headers.include?('title').should == true
+    headers.include?('owner').should == true
+
+    ["milestones", "loader_releases", "versions", "categories"].each do |check|
+      headers.should_not include check
+    end
+
+  end
+
+
+  it "should enable us to exclude certain associations in template .csv file " do
+
+    expected = result_file('project_plus_some_assoc_template_spec.csv')
+
+    gen = CsvGenerator.new(expected)
+
+    options = {:remove => [:milestones, :versions] }
+
+    gen.generate_with_associations(Project, options)
+
+    expect(File.exists?(expected)).to eq true#, "Failed to find expected result file #{expected}"
+
+    csv = CSV.read(expected)
+
+    headers = csv[0]
+
+    ["title", "loader_releases", "owner", "categories"].each do |check|
+      headers.should include check
+    end
+
+
+    ["milestones",  "versions", ].each do |check|
+      headers.should_not include check
+    end
+
+  end
+
+
+   it "should enable us to remove standard rails feilds from template .csv file " do
+
+    expected = result_file('project_plus_some_assoc_template_spec.csv')
+
+    gen = CsvGenerator.new(expected)
+
+    options = {:remove_rails => true}
+
+    gen.generate_with_associations(Project, options)
+
+    expect(File.exists?(expected)).to eq true#, "Failed to find expected result file #{expected}"
+
+    csv = CSV.read(expected)
+
+    headers = csv[0]
+
+    ["id", "updated_at", "created_at"].each do |check|
+      headers.should_not include check
+    end
+
+
+  end
+end

data/spec/csv_loader_spec.rb

@@ -3,29 +3,210 @@
 # Date ::     Aug 2011
 # License::   MIT
 #
-# Details::   Specs for Excel aspect of Active Record Loader
+# Details::   Specs for CSV aspect of Active Record Loader
 #
 require File.dirname(__FILE__) + '/spec_helper'
 
 require 'erb'
-require 'excel_loader'
+require 'csv_loader'
 
-describe 'CSV Loader' do
+include DataShift
 
-  before(:all) do
-    
-    # load our test model definitions - Project etc
-    require ifixture_file('test_model_defs')  
-       
-    db_connect( 'test_file' )    # , test_memory, test_mysql
-    migrate_up
-    @klazz = Project
+describe 'Csv Loader' do
+
+  before(:each) do
+    DataShift::MethodDictionary.clear
+
+    @method_mapper = DataShift::MethodMapper.new
   end
-  
+
+
   before(:each) do
-    MethodDictionary.clear
-    MethodDictionary.find_operators( @klazz )
-    MethodDictionary.find_operators( @assoc_klazz )
+
+    %w{category_001 category_002 category_003 category_004 category_005}.each do |cat|
+      Category.find_or_create_by(reference: cat)
+    end
+
+
+  end
+
+  it "should be able to create a new csv loader and load object" do
+    loader = CsvLoader.new(Project)
+
+    loader.load_object.should_not be_nil
+    loader.load_object.should be_is_a(Project)
+    expect(loader.load_object.new_record?).to eq true
+  end
+
+  it "should process a simple .csv spreedsheet" do
+
+    loader = CsvLoader.new(Project)
+
+    count = Project.count
+    loader.perform_load ifixture_file('csv/SimpleProjects.csv')
+
+    loader.loaded_count.should == (Project.count - count)
+  end
+
+  it "should process multiple associationss from single column" do
+
+    DataShift::MethodDictionary.find_operators( Category )
+
+    DataShift::MethodDictionary.build_method_details( Category )
+
+    Project.find_by_title('001').should be_nil
+    count = Project.count
+
+    loader = CsvLoader.new(Project)
+
+    loader.perform_load( ifixture_file('csv/ProjectsSingleCategories.csv') )
+
+    loader.loaded_count.should be > 3
+    loader.loaded_count.should == (Project.count - count)
+
+    {'001' => 2, '002' => 1, '003' => 3, '099' => 0 }.each do|title, expected|
+      project = Project.find_by_title(title)
+
+      project.should_not be_nil
+      #puts "#{project.inspect} [#{project.categories.size}]"
+
+      expect(project.categories.size).to eq expected
+    end
+  end
+
+  it "should process multiple associations in csv file" do
+
+    loader = CsvLoader.new(Project)
+
+    count = Project.count
+    loader.perform_load( ifixture_file('csv/ProjectsMultiCategories.csv' ))
+
+    loader.loaded_count.should == (Project.count - count)
+
+    {'004' => 3, '005' => 1, '006' => 0, '007' => 1 }.each do|title, expected|
+      project = Project.find_by_title(title)
+
+      project.should_not be_nil
+
+      expect(project.categories.size).to eq expected
+    end
+
+  end
+
+  it "should process multiple associations with lookup specified in column from excel spreedsheet" do
+
+    loader = CsvLoader.new(Project)
+
+    count = Project.count
+    loader.perform_load( ifixture_file('csv/ProjectsMultiCategoriesHeaderLookup.csv'))
+
+    expect(loader.loaded_count).to eq (Project.count - count)
+    loader.loaded_count.should > 3
+
+    {'004' => 4, '005' => 1, '006' => 0, '007' => 1 }.each do|title, expected|
+      project = Project.find_by_title(title)
+
+      project.should_not be_nil
+
+       expect(project.categories.size).to eq expected
+    end
+
   end
 
-end
+  it "should process excel spreedsheet with extra undefined columns" do
+    loader = CsvLoader.new(Project)
+    lambda {loader.perform_load( ifixture_file('csv/BadAssociationName.csv') ) }.should_not raise_error
+  end
+
+  it "should NOT process excel spreedsheet with extra undefined columns when strict mode" do
+    loader = CsvLoader.new(Project)
+    expect {loader.perform_load( ifixture_file('csv/BadAssociationName.csv'), :strict => true)}.to raise_error(MappingDefinitionError)
+  end
+
+  it "should raise an error when mandatory columns missing" do
+    loader = CsvLoader.new(Project)
+    expect {loader.perform_load(ifixture_file('csv/ProjectsMultiCategories.csv'), :mandatory => ['not_an_option', 'must_be_there'])}.to raise_error(DataShift::MissingMandatoryError)
+  end
+
+  it "should provide facility to set default values", :focus => true do
+    loader = CsvLoader.new(Project)
+
+    populator = loader.populator
+
+    populator.set_default_value('value_as_string', 'some default text' )
+    populator.set_default_value('value_as_double', 45.467 )
+    populator.set_default_value('value_as_boolean', true )
+
+    texpected = Time.now.to_s(:db)
+
+    populator.set_default_value('value_as_datetime', texpected )
+
+    #value_as_string	Value as Text	value as datetime	value_as_boolean	value_as_double	category
+
+    loader.perform_load(ifixture_file('csv/ProjectsSingleCategories.csv'))
+
+    p = Project.find_by_title( '099' )
+
+    p.should_not be_nil
+
+    p.value_as_string.should == 'some default text'
+    p.value_as_double.should == 45.467
+    p.value_as_boolean.should == true
+    p.value_as_datetime.to_s(:db).should == texpected
+
+    # expected: "2012-09-17 10:00:52"
+    # got: Mon Sep 17 10:00:52 +0100 2012 (using ==)
+
+    p_no_defs = Project.first
+
+    p_no_defs.value_as_string.should_not == 'some default text'
+    p_no_defs.value_as_double.should_not == 45.467
+    p_no_defs.value_as_datetime.should_not == texpected
+
+  end
+
+  it "should provide facility to set pre and post fix values" do
+    loader = CsvLoader.new(Project)
+
+    loader.populator.set_prefix('value_as_string', 'myprefix' )
+    loader.populator.set_postfix('value_as_string', 'my post fix' )
+
+    #value_as_string	Value as Text	value as datetime	value_as_boolean	value_as_double	category
+
+    loader.perform_load( ifixture_file('csv/ProjectsSingleCategories.csv'))
+
+    p = Project.find_by_title( '001' )
+
+    p.should_not be_nil
+
+    p.value_as_string.should == 'myprefixDemo stringmy post fix'
+  end
+
+  it "should provide facility to set default values via YAML configuration" do
+    loader = CsvLoader.new(Project)
+
+    loader.configure_from( ifixture_file('ProjectsDefaults.yml') )
+
+    loader.perform_load( ifixture_file('csv/ProjectsSingleCategories.csv') )
+
+    p = Project.find_by_title( '099' )
+
+    p.should_not be_nil
+
+    p.value_as_string.should == "Default Project Value"
+  end
+
+
+  it "should provide facility to over ride values via YAML configuration", :fail => true do
+    loader = CsvLoader.new(Project)
+
+    loader.configure_from( ifixture_file('ProjectsDefaults.yml') )
+
+
+    loader.perform_load( ifixture_file('csv/ProjectsSingleCategories.csv') )
+
+    Project.all.each {|p| p.value_as_double.should == 99.23546 }
+  end
+
+
+end