datashift 0.0.1

data/spec/db/migrate/20110803201325_create_test_bed.rb

@@ -0,0 +1,85 @@
+# Author ::   Tom Statter
+# Date ::     Aug 2011
+# License::   MIT
+#
+# Details::   Migration to create a test database that exercises all aspects of Active Record models
+#
+
+class CreateTestBed < ActiveRecord::Migration
+  
+  def self.up
+
+    # has many :milestones
+    create_table :projects do |t|
+      t.string   :title
+      t.string   :value_as_string
+      t.text     :value_as_text
+      t.boolean  :value_as_boolean, :default => false
+      t.datetime :value_as_datetime, :default => nil
+      t.integer  :value_as_integer, :default => 0
+      t.decimal  :value_as_double, :precision => 8, :scale => 2, :default => 0.0
+      t.timestamps
+    end
+
+    # belongs_to  :project, project => has_many
+    create_table :milestones do |t|
+      t.string     :name
+      t.datetime   :datetime, :default => nil
+      t.decimal    :cost, :precision => 8, :scale => 2, :default => 0.0
+      t.references :project
+      t.timestamps
+    end
+
+    # belongs_to  :project, project => has_one
+    create_table :owners do |t|
+      t.string   :name
+      t.references :project
+      t.timestamps
+    end
+
+    # has_belongs_to_many :project
+    create_table :categories do |t|
+      t.string   :reference
+      t.timestamps
+    end
+
+    # testing has_belongs_to_many (hence no id)
+    create_table :categories_projects, :id => false do |t|
+      t.references :category
+      t.references :project
+    end
+
+    create_table :versions do |t|
+      t.string   :name
+      t.timestamps
+    end
+
+    # testing project has_many release + versions :through
+    create_table :loader_releases do |t|
+      t.string   :name
+      t.references :project
+      t.references :version
+      t.timestamps
+    end
+
+
+    create_table :long_and_complex_table_linked_to_versions do |t|
+      t.references :version
+    end
+
+    create_table :empties do |t|
+    end
+
+  end
+
+  def self.down
+    drop_table :projects
+    drop_table :categories
+    drop_table :loader_releases
+    drop_table :versions
+    drop_table :categories_projectss
+    drop_table :milestones
+    drop_table :long_and_complex_table_linked_to_versions
+    drop_table :empties
+  end
+end

data/spec/excel_generator_spec.rb

@@ -0,0 +1,79 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2011
+# License::   MIT
+#
+# Details::   Specs for Excel aspect of Active Record Loader
+#
+require File.dirname(__FILE__) + '/spec_helper'
+
+if(Guards::jruby?)
+  require 'erb'
+  require 'excel_generator'
+
+  include DataShift
+
+  describe 'Excel Generator' do
+
+    before(:all) do
+      db_connect( 'test_file' )    # , test_memory, test_mysql
+
+      # load our test model definitions - Project etc
+      require File.join($DataShiftFixturePath, 'test_model_defs')  
+   
+      # handle migration changes or reset of test DB
+      migrate_up
+
+      db_clear()    # todo read up about proper transactional fixtures
+      results_clear()
+
+      @klazz = Project
+      @assoc_klazz = Category
+    end
+  
+    before(:each) do
+      MethodMapper.clear
+      MethodMapper.find_operators( @klazz )
+      MethodMapper.find_operators( @assoc_klazz )
+    end
+  
+    it "should be able to create a new excel generator" do
+      generator = ExcelGenerator.new( 'dummy.xls' )
+      
+      generator.should_not be_nil
+    end
+  
+    it "should generate template .xls file from model" do
+
+      expect = result_file('project_template_spec.xls')
+
+      gen = ExcelGenerator.new( expect )
+    
+      gen.generate(Project)
+ 
+      File.exists?(expect).should be_true
+      
+      puts "Can manually check file @ #{expect}"
+    end
+
+    it "should export a simple model to .xls spreedsheet" do
+
+      Project.create( :value_as_string	=> 'Value as Text', :value_as_boolean => true,	:value_as_double => 75.672)
+      #001 Demo string	blah blah	2011-02-14	1.00	320.00
+
+      expect= result_file('simple_export_spec.xls')
+
+      gen = ExcelGenerator.new(expect)
+
+      items = Project.all
+
+      gen.export(items)
+
+      File.exists?(expect).should be_true
+
+    end
+
+  end
+else
+  puts "WARNING: skipped excel_generator_spec : Requires JRUBY - JExcelFile requires JAVA"
+end # jruby

data/spec/excel_loader_spec.rb

@@ -0,0 +1,177 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2011
+# License::   MIT
+#
+# Details::   Specs for Excel aspect of Active Record Loader
+#
+require File.dirname(__FILE__) + '/spec_helper'
+
+if(Guards::jruby?)
+
+  require 'erb'
+  require 'excel_loader'
+
+  include DataShift
+
+  describe 'Excel Loader' do
+
+    before(:all) do
+      db_connect( 'test_file' )    # , test_memory, test_mysql
+
+      # load our test model definitions - Project etc
+      require File.join($DataShiftFixturePath, 'test_model_defs')  
+   
+      # handle migration changes or reset of test DB
+      migrate_up
+
+      db_clear()    # todo read up about proper transactional fixtures
+
+
+      @klazz = Project
+      @assoc_klazz = Category
+    end
+  
+    before(:each) do
+
+      Project.delete_all
+    
+      %w{category_001 category_002 category_003}.each do |cat|
+        @assoc_klazz.find_or_create_by_reference(cat)
+      end
+
+
+      MethodMapper.clear
+      MethodMapper.find_operators( @klazz )
+      MethodMapper.find_operators( @assoc_klazz )
+    end
+  
+    it "should be able to create a new excel loader and load object" do
+      loader = ExcelLoader.new( @klazz)
+    
+      loader.load_object.should_not be_nil
+      loader.load_object.should be_is_a(@klazz)
+      loader.load_object.new_record?.should be_true
+    end
+  
+    it "should process a simple .xls spreedsheet" do
+  
+      loader = ExcelLoader.new(@klazz)
+ 
+      count = @klazz.count
+      loader.perform_load( $DataShiftFixturePath + '/SimpleProjects.xls')
+  
+      loader.loaded_count.should == (@klazz.count - count)
+  
+    end
+
+    it "should process multiple associationss from single column" do
+
+      @klazz.find_by_title('001').should be_nil
+      count = @klazz.count
+
+      loader = ExcelLoader.new(@klazz)
+    
+      loader.perform_load( $DataShiftFixturePath + '/ProjectsSingleCategories.xls')
+
+      loader.loaded_count.should be > 3
+      loader.loaded_count.should == (@klazz.count - count)
+
+      {'001' => 2, '002' => 1, '003' => 3, '099' => 0 }.each do|title, expected|
+        project = @klazz.find_by_title(title)
+
+        project.should_not be_nil
+        puts "#{project.inspect} [#{project.categories.size}]"
+      
+        project.should have(expected).categories
+      end
+    end
+
+    it "should process multiple associations in excel spreedsheet" do
+  
+      loader = ExcelLoader.new(Project)
+  
+      count = Project.count
+      loader.perform_load( $DataShiftFixturePath + '/ProjectsMultiCategories.xls')
+  
+      loader.loaded_count.should == (Project.count - count)
+  
+      {'004' => 3, '005' => 1, '006' => 0, '007' => 1 }.each do|title, expected|
+        project = @klazz.find_by_title(title)
+  
+        project.should_not be_nil
+
+        project.should have(expected).categories
+      end
+  
+    end
+  
+    it "should process excel spreedsheet with extra undefined columns" do
+      loader = ExcelLoader.new(Project)
+      lambda {loader.perform_load( ifixture_file('BadAssociationName.xls') ) }.should_not raise_error
+    end
+
+    it "should NOT process excel spreedsheet with extra undefined columns when strict mode" do
+      loader = ExcelLoader.new(Project)
+      expect {loader.perform_load( ifixture_file('BadAssociationName.xls'), :strict => true)}.to raise_error(MappingDefinitionError)
+    end
+
+    it "should raise an error when mandatory columns missing" do
+      loader = ExcelLoader.new(Project)
+      expect {loader.perform_load($DataShiftFixturePath + '/ProjectsMultiCategories.xls', :mandatory => ['not_an_option', 'must_be_there'] )}.to raise_error(DataShift::MissingMandatoryError)
+    end
+
+    it "should provide facility to set default values", :focus => true do
+      loader = ExcelLoader.new(Project)
+      
+      loader.set_default_value('value_as_string', 'some default text' )
+      loader.set_default_value('value_as_double', 45.467 )
+      loader.set_default_value('value_as_boolean', true )
+      
+      texpected = Time.now.to_s(:db)
+      
+      loader.set_default_value('value_as_datetime', texpected )
+      
+      #value_as_string	Value as Text	value as datetime	value_as_boolean	value_as_double	category
+
+      loader.perform_load($DataShiftFixturePath + '/ProjectsSingleCategories.xls')
+      
+      p = Project.find_by_title( '099' )
+      
+      p.should_not be_nil
+      
+      p.value_as_string.should == 'some default text'
+      p.value_as_double.should == 45.467
+      p.value_as_boolean.should == true
+      p.value_as_datetime.should == texpected
+
+      p_no_defs = Project.first
+      
+      p_no_defs.value_as_string.should_not == 'some default text'
+      p_no_defs.value_as_double.should_not == 45.467
+      p_no_defs.value_as_datetime.should_not == texpected
+      
+    end
+    
+    it "should provide facility to set pre and post fix values", :focus => true do
+      loader = ExcelLoader.new(Project)
+      
+      loader.set_prefix('value_as_string', 'myprefix' )
+      loader.set_postfix('value_as_string', 'my post fix' )
+      
+      #value_as_string	Value as Text	value as datetime	value_as_boolean	value_as_double	category
+
+      loader.perform_load($DataShiftFixturePath + '/ProjectsSingleCategories.xls')
+      
+      p = Project.find_by_title( '001' )
+      
+      p.should_not be_nil
+      
+      p.value_as_string.should == 'myprefixDemo stringmy post fix'      
+    end
+    
+  end
+
+else
+  puts "WARNING: skipped excel_loader_spec : Requires JRUBY - JExcelFile requires JAVA"
+end # jruby

data/spec/file_definitions.rb

@@ -0,0 +1,141 @@
+# To change this template, choose Tools | Templates
+# and open the template in the editor.
+
+$:.unshift File.join(File.dirname(__FILE__),'..','lib')
+$:.unshift File.join(File.dirname(__FILE__),'..','lib', 'engine')
+
+require 'test/unit'
+require 'file_definitions'
+
+class File_definitions < Test::Unit::TestCase
+
+  def setup
+  end
+
+  def test_fields_string
+    klass = Object.const_set('B', Class.new)
+
+    klass.module_eval do
+      include FileDefinitions
+      create_field_definition 'a_string'
+
+      create_field_attr_accessors
+    end
+
+    x = B.new '33'
+    assert_equal '33', x.a_string
+
+  end
+
+
+  def test_fields_symbols
+    klass = Object.const_set('SymClass', Class.new)
+    klass.module_eval do
+      include FileDefinitions
+      create_field_definition  [:a_symbol, :b_symbol]
+
+      create_field_attr_accessors
+    end
+
+    assert SymClass.new.respond_to? :a_symbol
+    assert SymClass.new.respond_to? :b_symbol
+  end
+
+  def test_fields_strings
+    klass = Object.const_set('A', Class.new)
+    klass.module_eval do
+      include FileDefinitions
+      create_field_definition %w{abc def  ghi jkl}
+
+      create_field_attr_accessors
+    end
+
+    x = A.new
+
+    assert_equal ["abc", "def", "ghi", "jkl"], A.field_definition.sort
+
+    A::add_field( "xyz" )
+
+    line = '1,2,3,4, 5'
+    x = A.new(line)
+
+    assert x.respond_to? 'abc'
+    assert x.respond_to? 'abc='
+    assert x.respond_to? :jkl
+    assert_equal x.current_line, line
+    assert_equal '1', x.abc
+    assert_equal 1.0, x.abc.to_f
+    assert_equal 1,   x.abc.to_i
+    assert_equal '4', x.jkl
+    assert_equal ' 5', x.xyz
+
+  end
+
+  def test_fixed_strings
+
+    klass = Object.const_set('AFixed', Class.new)
+
+    klass.module_eval do
+      include FileDefinitions
+
+      create_fixed_definition( 'value' => (0..7), 'date' => (8..15), :ccy => (16..18) )
+
+      create_field_attr_accessors
+    end
+
+    assert AFixed.respond_to?('fixed_definition')
+    assert AFixed.respond_to?('field_definition')
+    assert AFixed.respond_to?('add_field')
+    assert AFixed.respond_to?('file_set_field_by_map')
+    assert AFixed.respond_to?('split_on')
+    assert AFixed.respond_to?('split_on_write')
+
+    x = AFixed.new('0123456719990113EUR')
+
+    assert x.respond_to?('value')
+    assert x.respond_to?(:date)
+    assert x.respond_to?('ccy')
+
+    assert AFixed.field_definition.include?('ccy')
+
+    assert_equal 3,  AFixed.field_definition.size
+    assert_equal 3,  AFixed.fixed_definition.keys.size
+
+    assert_equal  ["ccy", "date", "value"], AFixed.field_definition.sort
+
+    assert_equal AFixed.field_definition.sort,  AFixed.fixed_definition.keys.sort
+
+    assert_instance_of Range, AFixed.fixed_definition.values[0]
+
+    assert_equal x.current_line, '0123456719990113EUR'
+    assert_equal x.value, '01234567'
+    assert_equal x.date, '19990113'
+    assert_equal x.ccy, 'EUR'
+
+    assert x.respond_to?(:parse)
+
+    x.parse('9876543220100630USD')
+
+    assert_equal x.current_line, '9876543220100630USD'
+    assert_equal x.value, '98765432'
+    assert_equal x.date, '20100630'
+    assert_equal x.ccy, 'USD'
+
+  end
+
+  def test_bad_setup
+    klass = Object.const_set('ABadClass', Class.new)
+    begin
+      klass.module_eval do
+        include FileDefinitions
+
+        create_fixed_definition( 'abc' )
+
+        create_field_attr_accessors
+      end
+      flunk    # We should never get here
+    rescue => e
+      assert e
+    end
+  end
+end