datashift 0.0.1

data/spec/fixtures/config/database.yml

@@ -0,0 +1,25 @@
+<% adapter = 'sqlite3' %>
+<% adapter = 'jdbcsqlite3' if(Guards::jruby? ) %>
+
+test_mysql:
+  adapter: jdbcmysql
+  database: interact_test_db
+  username: test
+  password: test
+  host: localhost
+
+test_memory:
+  adapter: <%= adapter %>
+  database: :memory
+
+test_file:
+  adapter: <%= adapter %>
+  database: <%= File.join($DataShiftFixturePath, 'interact_models_db.sqlite') %>
+  encoding: utf8
+  host: localhost
+
+test_spree_standalone:
+  adapter: <%= adapter %>
+  database: <%= File.join($DataShiftFixturePath, 'interact_spree_db.sqlite') %>
+  encoding: utf8
+  host: localhost

data/spec/fixtures/negative/SpreeProdMiss1Mandatory.csv

@@ -0,0 +1,4 @@
+"Name","Description","Available On","     Price","CostPrice","product_properties","count_on_hand","Taxons","Option Types"
+"Demo Product for AR Loader","blah blah","2011-02-14",400.00,320.00,"name:test_pp_001",12,"Paintings","mime_type:jpeg, PDF,PNG"
+"Demo Excel Load via Jruby","less blah","2011-05-14",100.00,30.00,"name:test_pp_001",5,"Paintings","mime_type:jpeg, PDF,PNG"
+"Demo third row in future","more blah blah","2012-07-01",50.34,23.34,"name:test_pp_002,test_pp_003",23,"Drawings","mime_type:jpeg,PDF,PNG|print_type:black_white,colour"

data/spec/fixtures/negative/SpreeProdMissManyMandatory.csv

@@ -0,0 +1,4 @@
+"SKU","Description","Available On","CostPrice","product_properties","count_on_hand","Taxons","Option Types"
+"DEMO_001","blah blah","2011-02-14",320.00,"name:test_pp_001",12,"Paintings","mime_type:jpeg, PDF,PNG"
+"DEMO_002","less blah","2011-05-14",30.00,"name:test_pp_001",5,"Paintings","mime_type:jpeg, PDF,PNG"
+"DEMO_003","more blah blah","2012-07-01",23.34,"name:test_pp_002,test_pp_003",23,"Drawings","mime_type:jpeg,PDF,PNG|print_type:black_white,colour"

data/spec/fixtures/spree/SpreeProducts.csv

@@ -0,0 +1,4 @@
+"SKU","Name","Description","Available On","     Price","CostPrice","product_properties","Taxons","Variants","count_on_hand"
+"DEMO_001","Demo Product for AR Loader","blah blah","2011-02-14",399.99,320.00,"test_pp_001","Paintings|WaterColour","mime_type:jpeg,PDF,PNG","12|6|7"
+"DEMO_002","Demo Excel Load via Jruby","less blah","2011-05-14",100.00,30.00,"test_pp_003","Paintings|Landscape|Oils","mime_type:jpeg, PDF,PNG","5|4|2"
+"DEMO_003","Demo third row in future","more blah blah","2012-07-01",50.34,23.34,"test_pp_002|test_pp_003:Example free value|yet_another_property","Drawings","mime_type:jpeg,PDF,PNG|print_type:black_white,colour","12|4|7|12|18"

data/spec/fixtures/spree/SpreeProductsMultiColumn.csv

@@ -0,0 +1,4 @@
+"SKU","Name","Description","Available On","     Price","CostPrice","product_properties","product_properties","product_properties","Taxons","Taxons","Taxons","Option Types","Option Types","Variants","Variants","Variants","count_on_hand"
+"DEMO_001","Demo Product for AR Loader","blah blah","2011-02-14",399.99,320.00,"test_pp_001",,,"Paintings","WaterColour",,"mime_type:jpeg","mime_type:PDF","mime_type:PNG",,,"12|6|7"
+"DEMO_002","Demo Excel Load via Jruby","less blah","2011-05-14",100.00,30.00,"test_pp_003",,,"Paintings","Landscape","Oils","mime_type:jpeg","mime_type:PDF","mime_type:PNG",,,"5|4|2"
+"DEMO_003","Demo third row in future","more blah blah","2012-07-01",50.34,23.34,"test_pp_002","test_pp_003:Example free value","yet_another_property","Drawings",,,"mime_type:jpeg","mime_type:PDF","mime_type:PNG","print_type:black_white","print_type:colour","12|4|7|12|18"

data/spec/fixtures/spree/SpreeProductsSimple.csv

@@ -0,0 +1,4 @@
+"SKU","Name","Description","Available On","     Price","CostPrice","count_on_hand","Option Types"
+"SIMPLE_001","Simple Product for AR Loader","blah blah","2011-02-14",345.78,320.00,12,"mime_type"
+"SIMPLE_002","Simple Excel Load via Jruby","less blah","2011-05-14",100.00,30.00,5,"mime_type"
+"SIMPLE_003","Simple third row avail in future","more blah blah","2012-07-01",50.34,23.34,23,"mime_type|print_type"

data/spec/fixtures/spree/SpreeZoneExample.csv

@@ -0,0 +1,5 @@
+"Name","Description"
+"ROW","Rest of World"
+"UK","United Kingdom"
+"EUR","Europe"
+"NA","North America"

data/spec/fixtures/test_model_defs.rb

@@ -0,0 +1,57 @@
+
+class Project < ActiveRecord::Base
+
+  has_one  :owner
+
+  has_many :milestones
+
+  has_many :loader_releases
+  has_many :versions, :through => :loader_releases
+
+
+  #has_many :release_versions, :through => :loader_releases, :source => :versions
+
+  has_and_belongs_to_many :categories
+
+end
+
+class Owner < ActiveRecord::Base
+  belongs_to :project
+end
+
+class Milestone < ActiveRecord::Base
+  belongs_to :project
+  #validate the name, cost
+
+  delegate :title, :title=, :to => :project
+end
+
+# had_and_belongs to join table
+class Category < ActiveRecord::Base
+  has_and_belongs_to_many :projects
+end
+
+
+class Version < ActiveRecord::Base
+  has_many :releases
+
+  has_one :long_and_complex_table_linked_to_version
+end
+
+# Join Table with additional columns
+class LoaderRelease < ActiveRecord::Base
+  
+  belongs_to :project
+  belongs_to :version
+
+  #validate the name
+end
+
+class Empty < ActiveRecord::Base
+end
+
+# Join Table with additional columns
+class LongAndComplexTableLinkedToVersion < ActiveRecord::Base
+
+  belongs_to :version
+end

data/spec/loader_spec.rb

@@ -0,0 +1,121 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2011
+# License::   MIT
+#
+# Details::   Specs for base class Loader
+#
+require File.dirname(__FILE__) + '/spec_helper'
+
+require 'erb'
+
+describe 'Basic Loader' do
+
+  before(:all) do
+    db_connect( 'test_file' )    # , test_memory, test_mysql
+    
+    # load our test model definitions - Project etc
+    require File.join($DataShiftFixturePath, 'test_model_defs')  
+   
+    migrate_up
+    @klazz = Project
+  end
+  
+  before(:each) do
+    MethodMapper.clear
+    MethodMapper.find_operators( @klazz )
+  end
+  
+  it "should be able to create a new loader and load object" do
+    loader = LoaderBase.new( @klazz )
+
+    loader.load_object.should_not be_nil
+    loader.load_object.should be_is_a(@klazz)
+    loader.load_object.new_record?.should be_true
+  end
+
+  it "should process a string field against an assigment method detail" do
+
+    loader = LoaderBase.new(Project)
+
+    column = 'Value As String'
+    row = 'Another Lazy fox '
+
+    loader.find_and_process(column, row)
+
+    loader.load_object.value_as_string.should == row
+  end
+
+  it "should process a text field against an assigment method detail" do
+
+    loader = LoaderBase.new(Project)
+
+    column = :value_as_text
+    row = "Another Lazy fox\nJumped over something and bumped,\nHis head"
+
+    loader.find_and_process(column, row)
+
+    loader.load_object.value_as_text.should == row
+
+  end
+
+  it "should process a boolean field against an assigment method detail" do
+
+    loader = LoaderBase.new(Project)
+
+    column = :value_as_boolean
+    row = true
+
+    loader.find_and_process(column, row)
+
+    loader.load_object.value_as_boolean.should == row
+
+    row = 'false'
+
+    loader.find_and_process(column, row)
+
+    loader.load_object.value_as_boolean.should == false
+
+
+  end
+
+  it "should process a double field against an assigment operator" do
+  end
+
+  it "should process various date formats against a date assigment operator" do
+
+    loader = LoaderBase.new(Project)
+
+    column = :value_as_datetime
+
+    loader.find_and_process(column, Time.now)
+    loader.load_object.value_as_datetime.should_not be_nil
+
+    loader.find_and_process(column, "2011-07-23")
+    loader.load_object.value_as_datetime.should_not be_nil
+
+    loader.find_and_process(column, "Sat Jul 23 09:01:56 +0100 2011")
+    loader.load_object.value_as_datetime.should_not be_nil
+
+    loader.find_and_process(column,  Time.now.to_s(:db))
+    loader.load_object.value_as_datetime.should_not be_nil
+
+    loader.find_and_process(column,  "Jul 23 2011 23:02:59")
+    loader.load_object.value_as_datetime.should_not be_nil
+
+    if(Guards.jruby?)
+      loader.find_and_process(column,  "07/23/2011")    # dd/mm/YYYY
+      loader.load_object.value_as_datetime.should_not be_nil
+    end
+    
+    # bad casts - TODO - is this really an error needs raising ?
+    loader.find_and_process(column, "2011 07 23")
+    loader.load_object.value_as_datetime.should be_nil
+
+
+    loader.find_and_process(column,  "2011-23-07")
+    loader.load_object.value_as_datetime.should be_nil
+
+  end
+
+end

data/spec/method_mapper_spec.rb

@@ -0,0 +1,238 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2011
+# License::   MIT
+#
+# Details::   Specs for MethodMapper aspect of Active Record Loader
+#             MethodMapper provides the bridge between 'strings' e.g column headings
+#             and a classes different types of assignment operators
+#
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe 'Method Mapping' do
+
+  before(:all) do
+    db_connect( 'test_file' )    # , test_memory, test_mysql
+    migrate_up
+    @klazz = Project
+    @assoc_klazz = Milestone
+  end
+  
+  before(:each) do
+    MethodMapper.clear
+    MethodMapper.find_operators( @klazz )
+    MethodMapper.find_operators( @assoc_klazz )
+  end
+  
+  it "should populate method map for a given AR model" do
+
+    MethodMapper.has_many.should_not be_empty
+    MethodMapper.has_many[Project].should include('milestones')
+
+    MethodMapper.assignments.should_not be_empty
+    MethodMapper.assignments[Project].should include('id')
+    MethodMapper.assignments[Project].should include('value_as_string')
+    MethodMapper.assignments[Project].should include('value_as_text')
+
+    MethodMapper.belongs_to.should_not be_empty
+    MethodMapper.belongs_to[Project].should be_empty
+
+
+    MethodMapper.column_types.should be_is_a(Hash)
+    MethodMapper.column_types.should_not be_empty
+    MethodMapper.column_types[Project].size.should == Project.columns.size
+
+
+  end
+
+  it "should populate assigment members without the equivalent association names" do
+
+    # we should remove has-many & belongs_to from basic assignment set as they require a DB lookup
+    # or a Model.create call, not a simple assignment
+
+    MethodMapper.assignments_for(@klazz).should_not include( MethodMapper.belongs_to_for(@klazz) )
+    MethodMapper.assignments_for(@klazz).should_not include( MethodMapper.has_many_for(@klazz) )
+  end
+
+
+  it "should populate assignment operators for method details for different forms of a column name" do
+
+    [:value_as_string, 'value_as_string', "VALUE as_STRING", "value as string"].each do |format|
+
+      method_details = MethodMapper.find_method_detail( @klazz, format )
+
+      method_details.class.should == MethodDetail
+
+      method_details.name.should eq( format.to_s )
+
+      method_details.operator.should == 'value_as_string'
+      method_details.operator_for(:assignment).should == 'value_as_string'
+      
+      method_details.operator?('value_as_string').should be_true
+      method_details.operator?('blah_as_string').should be_false
+
+      method_details.operator_for(:belongs_to).should be_nil
+      method_details.operator_for(:has_many).should be_nil
+    end
+  end
+
+
+  # Note : Not all assignments will currently have a column type, for example
+  # those that are derived from a delegate_belongs_to
+
+  it "should populate column types for assignment operators in method details" do
+
+    [:value_as_string, 'value_as_string', "VALUE as_STRING", "value as string"].each do |format|
+
+      method_details = MethodMapper.find_method_detail( @klazz, format )
+
+      method_details.class.should == MethodDetail
+
+      method_details.col_type.should_not be_nil
+      method_details.col_type.name.should == 'value_as_string'
+      method_details.col_type.default.should == nil
+      method_details.col_type.sql_type.should include 'varchar(255)'   # db specific, sqlite
+      method_details.col_type.type.should == :string
+    end
+  end
+
+  it "should populate required Class for assignment operators based on column type" do
+
+    [:value_as_string, 'value_as_string', "VALUE as_STRING", "value as string"].each do |format|
+
+      method_details = MethodMapper.find_method_detail( Project, format )
+
+      method_details.operator_class_name.should == 'String'
+      method_details.operator_class.should be_is_a(Class)
+      method_details.operator_class.should == String
+    end
+
+  end
+
+  it "should populate belongs_to operator for method details for different forms of a column name" do
+
+    # milestone.project = project.id
+    [:project, 'project', "PROJECT", "prOJECt"].each do |format|
+
+      method_details = MethodMapper.find_method_detail( Milestone, format )
+
+      method_details.should_not be_nil
+
+      method_details.operator.should == 'project'
+      method_details.operator_for(:belongs_to).should == 'project'
+
+      method_details.operator_for(:assignment).should be_nil
+      method_details.operator_for(:has_many).should be_nil
+    end
+
+  end
+
+  it "should populate required Class for belongs_to operator method details" do
+
+    MethodMapper.find_operators( LoaderRelease )
+    MethodMapper.find_operators( LongAndComplexTableLinkedToVersion )
+
+    # release.project = project.id
+    [:project, 'project', "PROJECT", "prOJECt"].each do |format|
+
+      method_details = MethodMapper.find_method_detail( LoaderRelease, format )
+
+      method_details.operator_class_name.should == 'Project'
+      method_details.operator_class.should == Project
+    end
+
+
+    #LongAndComplexTableLinkedToVersion.version = version.id
+
+    [:version, "Version", "verSION"].each do |format|
+      method_details = MethodMapper.find_method_detail( LongAndComplexTableLinkedToVersion, format )
+
+      method_details.operator_type.should == :belongs_to
+
+      method_details.operator_class_name.should == 'Version'
+      method_details.operator_class.should == Version
+    end
+  end
+
+   it "should populate required Class for has_one operator method details" do
+
+    MethodMapper.find_operators( Version )
+
+    # version.long_and_complex_table_linked_to_version = LongAndComplexTableLinkedToVersion.create()
+
+    [:long_and_complex_table_linked_to_version, 'LongAndComplexTableLinkedToVersion', "Long And Complex_Table_Linked To  Version", "Long_And_Complex_Table_Linked_To_Version"].each do |format|
+      method_details = MethodMapper.find_method_detail( Version, format )
+
+      method_details.should_not be_nil
+
+      method_details.operator.should == 'long_and_complex_table_linked_to_version'
+      
+      method_details.operator_type.should == :has_one
+
+      method_details.operator_class_name.should == 'LongAndComplexTableLinkedToVersion'
+      method_details.operator_class.should == LongAndComplexTableLinkedToVersion
+    end
+  end
+
+
+  it "should find has_many operator for method details" do
+
+    [:milestones, "Mile Stones", 'mileSTONES', 'MileStones'].each do |format|
+
+      method_details = MethodMapper.find_method_detail( Project, format )
+
+      method_details.class.should == MethodDetail
+      
+      result = 'milestones'
+      method_details.operator.should == result
+      method_details.operator_for(:has_many).should == result
+
+      method_details.operator_for(:belongs_to).should be_nil
+      method_details.operator_for(:assignments).should be_nil
+    end
+
+  end
+
+ 
+  it "should return nil when non existent column name" do
+    ["On sale", 'on_sale'].each do |format|
+      detail = MethodMapper.find_method_detail( @klazz, format )
+
+      detail.should be_nil
+    end
+  end
+
+
+  it "should find a set of methods based on a list of column names" do
+
+    mapper = MethodMapper.new
+
+    [:value_as_string, 'value_as_string', "VALUE as_STRING", "value as string"].each do |column_name_format|
+
+      method_details = MethodMapper.find_method_detail( @klazz, column_name_format )
+
+      method_details.class.should == MethodDetail
+
+      method_details.col_type.should_not be_nil
+      method_details.col_type.name.should == 'value_as_string'
+      method_details.col_type.default.should == nil
+      method_details.col_type.sql_type.should include 'varchar(255)'   # db specific, sqlite
+      method_details.col_type.type.should == :string
+    end
+  end
+
+  it "should not by default map setter methods", :fail => true do
+    MethodMapper.assignments[Milestone].should_not include('title')
+  end
+  
+  it "should support reload and  inclusion of setter methods", :fail => true do
+
+    MethodMapper.assignments[Milestone].should_not include('title')
+        
+    MethodMapper.find_operators( Milestone, :reload => true, :instance_methods => true )
+     
+    # Milestone delegates :title to Project
+    MethodMapper.assignments[Milestone].should include('title')
+  end
+
+end