datashift 0.0.1

data/spec/spree_method_mapping_spec.rb

@@ -0,0 +1,215 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Summer 2011
+#
+# License::   MIT - Free, OpenSource
+#
+# Details::   Specification for Spree aspect of datashift gem.
+#
+#             Tests the method mapping aspect, such as that we correctly identify 
+#             Spree Product attributes and associations
+#             
+require File.dirname(__FILE__) + '/spec_helper'
+
+require 'spree_helper'
+require 'product_loader'
+
+include DataShift
+  
+describe 'SpreeLoader' do
+
+  before(:all) do
+
+    # we are not a Spree project, nor is it practical to externally generate
+    # a complete Spree application for testing so we implement a mini migrate/boot of our own
+    Spree.load()            # require Spree gems
+
+    # key to YAML db e.g  test_memory, test_mysql
+    db_connect( 'test_spree_standalone' )    
+
+    Spree.boot            # create a sort-of Spree app
+    
+    Spree.migrate_up      # create an sqlite Spree database on the fly
+
+    @klazz = Product
+
+    # Reset main tables - TODO should really purge properly, or roll back a transaction
+    [OptionType, OptionValue, Product, Property, Variant, Taxonomy, Taxon, Zone].each { |x| x.delete_all }
+  end
+
+  before(:each) do
+    MethodMapper.clear
+    MethodMapper.find_operators( @klazz )
+  end
+
+  
+  it "should populate operators for a Spree Product" do
+  
+    MethodMapper.has_many.should_not be_empty
+    MethodMapper.belongs_to.should_not be_empty
+    MethodMapper.assignments.should_not be_empty
+
+    assign = MethodMapper.assignments_for(@klazz)
+
+    assign.should include('available_on')   # Example of a simple column
+
+    MethodMapper.assignments[@klazz].should include('available_on')
+
+    has_many_ops = MethodMapper.has_many_for(@klazz)
+
+    has_many_ops.should include('properties')   # Product can have many properties
+
+    MethodMapper.has_many[@klazz].should include('properties')
+
+    btf = MethodMapper.belongs_to_for(@klazz)
+
+    btf.should include('tax_category')    # Example of a belongs_to assignment
+
+    MethodMapper.belongs_to[@klazz].should include('tax_category')
+
+    MethodMapper.column_types[@klazz].size.should == @klazz.columns.size
+  end
+
+
+  it "should find method details correctly for different forms of a column name" do
+
+    ["available On", 'available_on', "Available On", "AVAILABLE_ON"].each do |format|
+
+      method_details = MethodMapper.find_method_detail( @klazz, format )
+
+      method_details.operator.should == 'available_on'
+      method_details.operator_for(:assignment).should == 'available_on'
+
+      method_details.operator_for(:belongs_to).should be_nil
+      method_details.operator_for(:has_many).should be_nil
+
+      method_details.col_type.should_not be_nil
+      method_details.col_type.name.should == 'available_on'
+      method_details.col_type.default.should == nil
+      method_details.col_type.sql_type.should include 'datetime'   # works on mysql and sqlite
+      method_details.col_type.type.should == :datetime
+    end
+  end
+
+  it "should populate method details correctly for has_many forms of association name" do
+
+    MethodMapper.has_many[@klazz].should include('product_option_types')
+
+    ["product_option_types", "product option types", 'product Option_types', "ProductOptionTypes", "Product_Option_Types"].each do |format|
+      method_detail = MethodMapper.find_method_detail( @klazz, format )
+
+      method_detail.should_not be_nil
+
+      method_detail.operator_for(:has_many).should eq('product_option_types')
+      method_detail.operator_for(:belongs_to).should be_nil
+      method_detail.operator_for(:assignment).should be_nil
+    end
+  end
+
+
+  it "should populate method details correctly for assignment operators (none columns on #{@klazz})" do
+
+    MethodMapper.find_operators( @klazz, :reload => true, :instance_methods => true )
+
+    # Example of delegates i.e. cost_price column on Variant, delegated to Variant by Product
+
+    MethodMapper.assignments[@klazz].should include('cost_price')
+    MethodMapper.assignments[@klazz].should include('sku')
+
+
+    count_on_hand = MethodMapper.find_method_detail( @klazz, 'count on hand' )
+    count_on_hand.should_not be_nil
+    count_on_hand.operator.should == 'count_on_hand'
+
+    method = MethodMapper.find_method_detail( @klazz, 'sku' )
+    method.should_not be_nil
+    method.operator.should == 'sku'
+  end
+
+  it "should enable assignment via operators for none columns on #{@klazz}" do
+
+    MethodMapper.find_operators( @klazz, :reload => true, :instance_methods => true )
+
+    klazz_object = @klazz.new
+
+    klazz_object.should be_new_record
+
+    # we can use method details to populate a new AR object, essentailly same as
+    # klazz_object.send( count_on_hand.operator, 2)
+    count_on_hand = MethodMapper.find_method_detail( @klazz, 'count on hand' )
+
+    count_on_hand.assign( klazz_object, 2 )
+    klazz_object.count_on_hand.should == 2
+
+    count_on_hand.assign( klazz_object, 5 )
+    klazz_object.count_on_hand.should == 5
+
+    method = MethodMapper.find_method_detail( @klazz, 'sku' )
+    method.should_not be_nil
+
+    method.operator.should == 'sku'
+
+    method.assign( klazz_object, 'TEST_SK 001')
+    klazz_object.sku.should == 'TEST_SK 001'
+
+  end
+
+  it "should enable assignment to has_many association on new object" do
+ 
+    method_detail = MethodMapper.find_method_detail( @klazz, 'taxons' )
+
+    method_detail.operator.should == 'taxons'
+
+    klazz_object = @klazz.new
+
+    klazz_object.taxons.size.should == 0
+
+    # NEW ASSOCIATION ASSIGNMENT
+
+    # assign via the send operator directly on load object
+    klazz_object.send( method_detail.operator ) << Taxon.new
+
+    klazz_object.taxons.size.should == 1
+
+    klazz_object.send( method_detail.operator ) << [Taxon.new, Taxon.new]
+    klazz_object.taxons.size.should == 3
+
+    # Use generic assignment on method detail - expect has_many to use << not =
+    method_detail.assign( klazz_object, Taxon.new )
+    klazz_object.taxons.size.should == 4
+
+    method_detail.assign( klazz_object, [Taxon.new, Taxon.new])
+    klazz_object.taxons.size.should == 6
+  end
+
+  it "should enable assignment to has_many association using existing objects" do
+
+    MethodMapper.find_operators( @klazz )
+
+    method_detail = MethodMapper.find_method_detail( @klazz, 'product_properties' )
+
+    method_detail.operator.should == 'product_properties'
+
+    klazz_object = @klazz.new
+
+    ProductProperty.new(:property => @prop1)
+
+    # NEW ASSOCIATION ASSIGNMENT
+    klazz_object.send( method_detail.operator ) << ProductProperty.new
+
+    klazz_object.product_properties.size.should == 1
+
+    klazz_object.send( method_detail.operator ) << [ProductProperty.new, ProductProperty.new]
+    klazz_object.product_properties.size.should == 3
+
+    # Use generic assignment on method detail - expect has_many to use << not =
+    method_detail.assign( klazz_object, ProductProperty.new(:property => @prop1) )
+    klazz_object.product_properties.size.should == 4
+
+    method_detail.assign( klazz_object, [ProductProperty.new(:property => @prop2), ProductProperty.new(:property => @prop3)])
+    klazz_object.product_properties.size.should == 6
+
+  end
+
+  
+end

data/tasks/config/seed_fu_product_template.erb

@@ -0,0 +1,15 @@
+p = Product.seed( :name ) do |s|
+  s.name =  <%= @name %>
+  s.available_on =  '2009-09-01 09:00:00.0'
+  s.meta_keywords =  ['training', 'training']
+  s.meta_description =  ""
+  s.description =  '<%= @description %>'
+  s.price = 0.00
+  s.sku = '<%= @sku %>'
+
+  s.is_physical    = false
+  s.is_private     = false
+
+  s.append_association :taxons, Taxon.find_by_name( 'Training' )
+
+end

data/tasks/config/tidy_config.txt

@@ -0,0 +1,13 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2010
+#
+# License::   Free, OpenSource... MIT ?
+
+# This is the best effort I've found so far to reduce the amount of MS cruft
+# to absolute minimum
+# ... but unfortunately these tags will NOT BE REMOVED completely - manual cleanup still required
+
+# TODO - add another ruby parse layer to strip these out completely
+
+new-empty-tags: o:smarttagtype, st1:placename, st1:place, st1:placetype

data/tasks/db_tasks.rake

@@ -0,0 +1,65 @@
+# Author ::   Tom Statter
+# Date ::     Mar 2011
+#
+# License::   The MIT License (Free and OpenSource)
+#
+# About::     Additional Rake tasks useful when testing seeding DB via DataShift
+#
+namespace :autotelik do
+
+  namespace :db  do
+
+    SYSTEM_TABLE_EXCLUSION_LIST = ['schema_migrations']
+
+    desc "Purge the current database"
+    task :purge, :exclude_system_tables, :needs => [:environment] do |t, args|
+      require 'highline/import'
+
+      if(RAILS_ENV == 'production')
+        agree("WARNING: In Production database, REALLY PURGE ? [y]:")
+      end
+
+      config = ActiveRecord::Base.configurations[RAILS_ENV || 'development']
+      case config['adapter']
+      when "mysql", "jdbcmysql"
+        ActiveRecord::Base.establish_connection(config)
+        ActiveRecord::Base.connection.tables.each do |table|
+          next if(args[:exclude_system_tables] && SYSTEM_TABLE_EXCLUSION_LIST.include?(table) )
+          puts "purging table: #{table}"
+          ActiveRecord::Base.connection.execute("TRUNCATE #{table}")
+        end
+      when "sqlite","sqlite3"
+        dbfile = config["database"] || config["dbfile"]
+        File.delete(dbfile) if File.exist?(dbfile)
+      when "sqlserver"
+        dropfkscript = "#{config["host"]}.#{config["database"]}.DP1".gsub(/\\/,'-')
+        `osql -E -S #{config["host"]} -d #{config["database"]} -i db\\#{dropfkscript}`
+        `osql -E -S #{config["host"]} -d #{config["database"]} -i db\\#{RAILS_ENV}_structure.sql`
+      when "oci", "oracle"
+        ActiveRecord::Base.establish_connection(config)
+        ActiveRecord::Base.connection.structure_drop.split(";\n\n").each do |ddl|
+          ActiveRecord::Base.connection.execute(ddl)
+        end
+      when "firebird"
+        ActiveRecord::Base.establish_connection(config)
+        ActiveRecord::Base.connection.recreate_database!
+      else
+        raise "Task not supported by '#{config["adapter"]}'"
+      end
+    end
+ 
+    desc "Clear database and optional directories such as assets, then run db:seed"
+    task :seed_again, :assets, :needs => [:environment] do |t, args|
+
+      Rake::Task['autotelik:db:purge'].invoke( true )   # i.e ENV['exclude_system_tables'] = true
+
+      if(args[:assets])
+        assets = "#{Rails.root}/public/assets"
+        FileUtils::rm_rf(assets) if(File.exists?(assets))
+      end
+
+      Rake::Task['db:seed'].invoke
+    end
+  
+  end # db
+end # autotelik

data/tasks/excel_generator.rake

@@ -0,0 +1,79 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2011
+# License::   MIT.
+#
+# REQUIRES:   JRuby
+#
+# Usage::
+#
+#  In Rakefile:
+#
+#     require 'datashift'
+#
+#     DataShift::load_tasks
+#
+#  Cmd Line:
+#
+# => jruby -S rake datashift:generate:excel model=<active record class> result=<output_template.xls>
+#
+namespace :datashift do
+  namespace :generate do
+
+    desc "Generate a template .xls (Excel) file for a model"
+  
+    task :excel, [:model, :result] => [:environment] do |t, args|
+
+      require 'excel_generator'
+      
+      # in familiar ruby style args seems to have been become empty using this new style for rake 0.9.2
+      #  whatever format i try, on both Win and OSX .. so had to revert back to ENV
+      model = args[:model] || ENV['model']
+      result = args[:result] || ENV['result']
+    
+      raise "USAGE: jruby -S rake datashift:generate:excel model=<Class> result=<file.xls>" unless(result)
+      raise "ERROR: Cannot process without AR Model - please supply model=<Class>" unless(model)
+
+      begin
+        klass = Kernel.const_get(model)
+      rescue NameError
+        raise "ERROR: No such AR Model found - check valid model supplied via model=<Class>"
+      end
+
+      gen = DataShift::ExcelGenerator.new(result)
+
+      gen.generate(klass)
+    end
+
+  end
+
+  namespace :export do
+
+    desc "Export active record data to .xls (Excel) file"
+
+    task :excel, [:model, :result] => [:environment] do |t, args|
+
+      require 'excel_generator'
+
+      # in familiar ruby style args seems to have been become empty using this new style for rake 0.9.2
+      #  whatever format i try, on both Win and OSX .. so had to revert back to ENV
+      model = ENV['model']
+      result = ENV['result']
+
+      raise "USAGE: jruby -S rake datashift:gen:excel model=<Class> result=<file.xls>" unless(result)
+      raise "ERROR: Cannot process without AR Model - please supply model=<Class>" unless(model)
+
+      begin
+        klass = Kernel.const_get(model)
+      rescue NameError
+        raise "ERROR: No such AR Model found - check valid model supplied via model=<Class>"
+      end
+
+      gen = DataShift::ExcelGenerator.new(result)
+      
+      gen.export(klass.all)
+    end
+
+  end
+
+end

data/tasks/file_tasks.rake

@@ -0,0 +1,37 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Feb 2011
+# License::   MIT
+#
+# Usage::     rake datashift:file_rename input=/blah image_load input=path_to_images
+#
+namespace :datashift do
+
+  desc "copy or mv a folder of files, consistently renaming in the process"
+  task :file_rename, :input, :offset, :prefix, :width, :commit, :mv do |t, args|
+    raise "USAGE: rake file_rename input='C:\blah' [offset=n prefix='str' width=n]" unless args[:input] && File.exists?(args[:input])
+    width = args[:width] || 2
+
+    action = args[:mv] ? 'mv' : 'cp'
+
+    cache = args[:input]
+
+    if(File.exists?(cache) )
+      puts "Renaming files from #{cache}"
+      Dir.glob(File.join(cache, "*")) do |name|
+        path, base_name = File.split(name)
+        id = base_name.slice!(/\w+/)
+
+        id = id.to_i + args[:offset].to_i if(args[:offset])
+        id = "%0#{width}d" % id.to_i if(args[:width])
+        id = args[:prefix] + id.to_s if(args[:prefix])
+
+        destination = File.join(path, "#{id}#{base_name}")
+        puts "ACTION: #{action} #{name} #{destination}"
+
+        File.send( action, name, destination) if args[:commit]
+      end
+    end
+  end
+
+end

data/tasks/import/csv.rake

@@ -0,0 +1,50 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2011
+# License::   MIT
+#
+# Usage::
+#
+#  In Rakefile:
+#
+#     require 'datashift'
+#
+#     DataShift::load_tasks
+#
+#  Cmd Line:
+#
+# => jruby -S rake datashift:import:csv model=<active record class> input=<file.csv>
+#
+namespace :datashift do
+
+  namespace :import do
+
+    desc "Populate a model's table in db with data from CSV file"
+    task :csv, [:model, :loader, :input, :verbose] => [:environment] do |t, args|
+
+      # in familiar ruby style args seems to have been become empty with rake 0.9.2 whatever i try
+      # so had to revert back to ENV
+      model = ENV['model']
+      input = ENV['input']
+   
+      raise "USAGE: rake datashift:import:csv input=file.csv model=<Class>" unless(input)
+      raise "ERROR: Cannot process without AR Model - please supply model=<Class>" unless(model)
+      raise "ERROR: Could not find csv file #{args[:input]}" unless File.exists?(input)
+
+      require 'csv_loader'
+      
+      begin
+        klass = Kernel.const_get(model)
+      rescue NameError
+        raise "ERROR: No such AR Model found - check valid model supplied via model=<Class>"
+      end
+
+      puts "INFO: Using CSV loader"
+    
+      loader = DataShift::CsvLoader.new(klass)
+
+      loader.perform_load(input)
+    end
+  end
+  
+end