ar_loader 0.0.4

data/lib/loaders/loader_base.rb

@@ -0,0 +1,61 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2010
+# License::   MIT
+#
+class LoaderBase
+
+  attr_accessor :load_object, :value
+
+  # Enable single column (association) to contain multiple name/value sets in default form :
+  #   Name1:value1, value2|Name2:value1, value2, value3|Name3:value1, value2
+  #
+  # E.G.
+  #   Row for association could have a name (Size/Colour/Sex) with a set of values,
+  #   and this combination can be expressed multiple times :
+  #   Size:small,medium,large|Colour:red, green|Sex:Female
+
+  @@name_value_delim  = ':'
+  @@multi_value_delim = ','
+  @@multi_assoc_delim = '|'
+
+  def self.set_name_value_delim(x)  @@name_value_delim = x; end
+  def self.set_multi_value_delim(x) @@multi_value_delim = x; end
+  def self.set_multi_assoc_delim(x) @@multi_assoc_delim = x; end
+
+  def initialize(object)
+    @load_object = object
+  end
+
+  # What process a value string from a column.
+  # Assigning value(s) to correct association on @load_object.
+  # Method map represents a column from a file and it's correlated AR associations.
+  # Value string which may contain multiple values for a collection association.
+  # 
+  def process(method_map, value)
+    @value = value
+    
+    if(method_map.has_many && method_map.has_many_class && @value)
+      # The Generic handler for Associations
+      # The actual class of the association so we can find_or_create on it
+      assoc_class = method_map.has_many_class
+
+      puts "Processing Association: #{assoc_class} : #{@value}"
+
+      @value.split(@@multi_assoc_delim).collect do |lookup|
+        # TODO - Don't just rely on 'name' but try different finds as per MethodMappe::insistent_belongs_to ..
+        x = assoc_class.find(:first, :conditions => ['lower(name) LIKE ?', "%#{lookup.downcase}%"])
+        unless x
+          puts "WARNING: #{lookup} in #{assoc_class} NOT found - Not added to #{@load_object.class}"
+          next
+        end
+        @load_object.send( method_map.has_many ) << x
+        @load_object.save
+      end
+    else
+      # Nice n simple straight assignment to a column variable
+      method_map.assign(@load_object, @value) unless method_map.has_many
+    end
+  end
+
+end

data/lib/loaders/spree/image_loader.rb

@@ -0,0 +1,47 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Jan 2011
+# License::   MIT. Free, Open Source.
+#
+require 'loader_base'
+
+class ImageLoader < LoaderBase
+
+  def initialize(image = nil)
+    obj = image || Image.create
+    super( obj )
+    raise "Failed to create Image for loading" unless @load_object
+  end
+
+  def refresh
+    @load_object = Image.create
+  end
+
+  # Note the Spree Image model sets default storage path to
+  # => :path => ":rails_root/public/assets/products/:id/:style/:basename.:extension"
+
+  def process( image_path, record = nil)
+
+    unless File.exists?(image_path)
+      puts "ERROR : Invalid Path"
+      return
+    end
+
+    alt = (record and record.respond_to? :name) ? record.name : ""
+
+    @load_object.alt = alt
+
+    begin
+      @load_object.attachment = File.new(image_path, "r")
+    rescue => e
+      puts e.inspect
+      puts "ERROR : Failed to read image #{image_path}"
+      return
+    end
+
+    @load_object.attachment.reprocess!
+    @load_object.viewable = record if record
+
+    puts @load_object.save ? "Success: Uploaded Image: #{@load_object.inspect}" : "ERROR : Problem saving to DB Image: #{@load_object}"
+  end
+end

data/lib/loaders/spree/product_loader.rb

@@ -0,0 +1,93 @@
+# Copyright:: (c) Autotelik Media Ltd 2010
+# Author ::   Tom Statter
+# Date ::     Aug 2010
+# License::   MIT ?
+#
+# Details::   Specific over-rides/additions to general loader to
+#             support Spree Products
+#
+require 'loader_base'
+
+class ProductLoader < LoaderBase
+
+  def initialize(product = nil)
+    prod = product || Product.create
+    super( prod )
+    raise "Failed to create Product for loading" unless @load_object
+  end
+
+  # What process a value string from a column, assigning value(s) to correct association on Product.
+  # Method map represents a column from a file and it's correlated Product association.
+  # Value string which may contain multiple values for a collection association.
+  # Product to assign that value to.
+  def process( method_map, value)
+    @value = value
+
+    #puts "DEBUG : process #{method_map.inspect} : #{value.inspect}"
+    # Special case for OptionTypes as it's two stage process
+    # First add the possible option_types to Product, then we are able
+    # to define Variants on those options.
+    
+    if(method_map.name == 'option_types' && @value)
+
+      option_types = @value.split(@@multi_assoc_delim)
+      option_types.each do |ostr|
+        oname, value_str = ostr.split(@@name_value_delim)
+        option_type = OptionType.find_or_create_by_name(oname)
+        unless option_type
+          puts "WARNING: OptionType #{oname} NOT found - Not set Product"
+          next
+        end
+
+        @load_object.option_types << option_type unless @load_object.option_types.include?(option_type)
+
+        # Now get the value(s) for the option e.g red,blue,green for OptType 'colour'
+        ovalues = value_str.split(',')
+        ovalues.each_with_index do |ovname, i|
+          ovname.strip!
+          ov = OptionValue.find_by_name(ovname)
+          if ov
+            object = Variant.new( :sku => "#{@load_object.sku}_#{i}", :price => @load_object.price, :available_on => @load_object.available_on)
+            #puts "DEBUG: Create New Variant: #{object.inspect}"
+            object.option_values << ov
+            @load_object.variants << object
+          else
+            puts "WARNING: Option #{ovname} NOT FOUND - No Variant created"
+          end
+        end
+      end
+
+    # Special case for ProductProperties since it can have additional value applied.
+    # A list of Properties with a optional Value - supplied in form :
+    #   Property:value|Property2:value|Property3:value
+    #
+    elsif(method_map.name == 'product_properties' && @value)
+
+      property_list = @value.split(@@multi_assoc_delim)
+
+      property_list.each do |pstr|
+        pname, pvalue = pstr.split(@@name_value_delim)
+        property = Property.find_by_name(pname)
+        unless property
+          puts "WARNING: Property #{pname} NOT found - Not set Product"
+          next
+        end
+        @load_object.product_properties << ProductProperty.create( :property => property, :value => pvalue)
+      end
+
+    elsif(method_map.name == 'count_on_hand' && @load_object.variants.size > 0 &&
+          @value.is_a?(String) && @value.include?(@@multi_assoc_delim))
+      # Check if we processed Option Types and assign count per option
+      values = @value.split(@@multi_assoc_delim)
+      if(@load_object.variants.size == values.size)
+        @load_object.variants.each_with_index {|v, i| v.count_on_hand == values[i] }
+      else
+        puts "WARNING: Count on hand entries does not match number of Variants"
+      end
+
+    else
+      super(method_map, value)
+    end
+
+  end
+end

data/lib/to_b.rb

@@ -0,0 +1,24 @@
+class Object
+  def to_b
+    case self
+    when true, false: self
+    when nil: false
+    else
+      to_i != 0
+    end
+  end
+end
+
+class String
+  TRUE_REGEXP = /^(yes|true|on|t|1|\-1)$/i.freeze
+  FALSE_REGEXP = /^(no|false|off|f|0)$/i.freeze
+
+  def to_b
+    case self
+    when TRUE_REGEXP: true
+    when FALSE_REGEXP: false
+    else
+      to_i != 0
+    end
+  end
+end

data/spec/excel_loader_spec.rb

@@ -0,0 +1,138 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe 'ExcelLoader' do
+
+  before do
+    @klazz = Product
+    MethodMapper.clear
+  end
+  
+  it "should populate operators for a given AR model" do
+    MethodMapper.find_operators( @klazz )
+
+    MethodMapper.has_many.should_not be_empty
+    MethodMapper.assignments.should_not be_empty
+
+    hmf = MethodMapper.has_many_for(@klazz)
+    arf = MethodMapper.assignments_for(@klazz)
+
+    (hmf & arf).should_not be_empty       # Associations provide << or =
+
+    hmf.should include('properties')
+    arf.should include('count_on_hand')   # example of a column
+    arf.should include('cost_price')      # example of delegated assignment (available through Variant)
+
+    MethodMapper.column_types.should be_is_a(Hash)
+    MethodMapper.column_types.should_not be_empty
+
+    MethodMapper.column_type_for(@klazz, 'count_on_hand').should_not be_nil
+  end
+
+  it "should populate operators respecting unique option" do
+    MethodMapper.find_operators( @klazz, :unique => true )
+
+    hmf = MethodMapper.has_many_for(@klazz)
+    arf = MethodMapper.assignments_for(@klazz)
+
+    (hmf & arf).should be_empty
+  end
+
+  it "should populate assignment method and col type for different forms of a column name" do
+
+    MethodMapper.find_operators( @klazz )
+
+    ["Count On hand", 'count_on_hand', "Count OnHand", "COUNT ONHand"].each do |format|
+      mmap = MethodMapper.determine_calls( @klazz, format )
+
+      mmap.class.should == MethodDetail
+
+      mmap.assignment.should == 'count_on_hand='
+      mmap.has_many.should be_nil
+
+      mmap.col_type.should_not be_nil
+      mmap.col_type.name.should == 'count_on_hand'
+      mmap.col_type.default.should == 0
+      mmap.col_type.sql_type.should == 'int(10)'
+      mmap.col_type.type.should == :integer
+    end
+  end
+
+  it "should populate both methods for different forms of an association name" do
+
+    MethodMapper.find_operators( @klazz )
+    ["product_option_types", "product option types", 'product Option_types', "ProductOptionTypes", "Product_Option_Types"].each do |format|
+      mmap = MethodMapper.determine_calls( @klazz, format )
+
+      mmap.assignment.should == 'product_option_types='
+      mmap.has_many.should   == 'product_option_types'
+
+      mmap.col_type.should be_nil
+    end
+  end
+
+
+  it "should not populate anything when  non existent column name" do
+    ["On sale", 'on_sale'].each do |format|
+      mmap = MethodMapper.determine_calls( @klazz, format )
+
+      mmap.class.should == MethodDetail
+      mmap.assignment.should be_nil
+      mmap.has_many.should be_nil
+      mmap.col_type.should be_nil
+    end
+  end
+
+  it "should enable correct assignment and sending of a value to AR model" do
+
+    MethodMapper.find_operators( @klazz )
+    
+    mmap = MethodMapper.determine_calls( @klazz, 'count on hand' )
+    mmap.assignment.should == 'count_on_hand='
+
+    x = @klazz.new
+
+    x.should be_new_record
+
+    x.send( mmap.assignment, 2 )
+    x.count_on_hand.should == 2
+    x.on_hand.should == 2     # helper method I know looks at same thing
+
+    mmap = MethodMapper.determine_calls( @klazz, 'SKU' )
+    mmap.assignment.should == 'sku='
+    x.send( mmap.assignment, 'TEST_SK 001' )
+    x.sku.should == 'TEST_SK 001'
+  end
+
+  it "should enable correct assignment and sending of association to AR model" do
+
+    MethodMapper.find_operators( @klazz )
+
+    mmap = MethodMapper.determine_calls( @klazz, 'taxons' )
+    mmap.has_many.should == 'taxons'
+
+    x = @klazz.new
+
+    # NEW ASSOCIATION ASSIGNMENT  v.,mn
+    x.send( mmap.has_many ) << Taxon.new
+    x.taxons.size.should == 1
+
+    x.send( mmap.has_many ) << [Taxon.new, Taxon.new]
+    x.taxons.size.should == 3
+
+    # EXISTING ASSOCIATIONS
+    x = Product.find :first
+
+    t = Taxonomy.find_or_create_by_name( 'BlahSpecTest' )
+
+    if x
+      sz = x.taxons.size
+      x.send(mmap.has_many) << t.root
+      x.taxons.size.should == sz + 1
+    else
+      puts "WARNING : Test not run could not find any Test Products"
+    end
+
+  end
+
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,37 @@
+unless defined? SPREE_ROOT
+  ENV["RAILS_ENV"] = "test"
+  case
+  when ENV["SPREE_ENV_FILE"]
+    require ENV["SPREE_ENV_FILE"]
+  when File.dirname(__FILE__) =~ %r{vendor/SPREE/vendor/extensions}
+    require "#{File.expand_path(File.dirname(__FILE__) + "/../../../../../../")}/config/environment"
+  else
+    require "#{File.expand_path(File.dirname(__FILE__) + "/../../../../")}/config/environment"
+  end
+end
+require "#{SPREE_ROOT}/spec/spec_helper"
+
+if File.directory?(File.dirname(__FILE__) + "/scenarios")
+  Scenario.load_paths.unshift File.dirname(__FILE__) + "/scenarios"
+end
+if File.directory?(File.dirname(__FILE__) + "/matchers")
+  Dir[File.dirname(__FILE__) + "/matchers/*.rb"].each {|file| require file }
+end
+
+Spec::Runner.configure do |config|
+  # config.use_transactional_fixtures = true
+  # config.use_instantiated_fixtures  = false
+  # config.fixture_path = RAILS_ROOT + '/spec/fixtures'
+
+  # You can declare fixtures for each behaviour like this:
+  #   describe "...." do
+  #     fixtures :table_a, :table_b
+  #
+  # Alternatively, if you prefer to declare them only once, you can
+  # do so here, like so ...
+  #
+  #   config.global_fixtures = :table_a, :table_b
+  #
+  # If you declare global fixtures, be aware that they will be declared
+  # for all of your examples, even those that don't use them.
+end

data/tasks/db_tasks.rake

@@ -0,0 +1,65 @@
+# Author ::   Tom Statter
+# Date ::     Mar 2011
+#
+# License::   The MIT License (Free and OpenSource)
+#
+# About::     Additional Rake tasks useful when testing seeding DB via ARLoader
+#
+namespace :autotelik do
+
+  namespace :db  do
+
+    SYSTEM_TABLE_EXCLUSION_LIST = ['schema_migrations']
+
+    desc "Purge the current database"
+    task :purge, :exclude_system_tables, :needs => [:environment] do |t, args|
+      require 'highline/import'
+
+      if(RAILS_ENV == 'production')
+        agree("WARNING: In Production database, REALLY PURGE ? [y]:")
+      end
+
+      config = ActiveRecord::Base.configurations[RAILS_ENV || 'development']
+      case config['adapter']
+      when "mysql", "jdbcmysql"
+        ActiveRecord::Base.establish_connection(config)
+        ActiveRecord::Base.connection.tables.each do |table|
+          next if(args[:exclude_system_tables] && SYSTEM_TABLE_EXCLUSION_LIST.include?(table) )
+          puts "purging table: #{table}"
+          ActiveRecord::Base.connection.execute("TRUNCATE #{table}")
+        end
+      when "sqlite","sqlite3"
+        dbfile = config["database"] || config["dbfile"]
+        File.delete(dbfile) if File.exist?(dbfile)
+      when "sqlserver"
+        dropfkscript = "#{config["host"]}.#{config["database"]}.DP1".gsub(/\\/,'-')
+        `osql -E -S #{config["host"]} -d #{config["database"]} -i db\\#{dropfkscript}`
+        `osql -E -S #{config["host"]} -d #{config["database"]} -i db\\#{RAILS_ENV}_structure.sql`
+      when "oci", "oracle"
+        ActiveRecord::Base.establish_connection(config)
+        ActiveRecord::Base.connection.structure_drop.split(";\n\n").each do |ddl|
+          ActiveRecord::Base.connection.execute(ddl)
+        end
+      when "firebird"
+        ActiveRecord::Base.establish_connection(config)
+        ActiveRecord::Base.connection.recreate_database!
+      else
+        raise "Task not supported by '#{config["adapter"]}'"
+      end
+    end
+ 
+    desc "Clear database and optional directories such as assets, then run db:seed"
+    task :seed_again, :assets, :needs => [:environment] do |t, args|
+
+      Rake::Task['autotelik:db:purge'].invoke( true )   # i.e ENV['exclude_system_tables'] = true
+
+      if(args[:assets])
+        assets = "#{Rails.root}/public/assets"
+        FileUtils::rm_rf(assets) if(File.exists?(assets))
+      end
+
+      Rake::Task['db:seed'].invoke
+    end
+  
+  end # db
+end # autotelik

data/tasks/excel_loader.rake

@@ -0,0 +1,101 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Feb 2011
+# License::   TBD. Free, Open Source. MIT ?
+#
+# REQUIRES:   JRuby
+#
+# Usage from rake : jruby -S rake excel_loader input=<file.xls>
+#
+# e.g.  => jruby -S rake excel_load input=vendor\extensions\autotelik\fixtures\ExampleInfoWeb.xls
+#       => jruby -S rake excel_load input=C:\MyProducts.xls verbose=true
+#
+namespace :autotelik do
+
+  desc "Populate AR model's table with data stored in Excel"
+  task :excel_load, :klass, :input, :verbose, :sku_prefix, :needs => :environment do |t, args|
+
+    raise "USAGE: jruby -S rake excel_load input=excel_file.xls" unless args[:input]
+    raise "ERROR: Cannot process without AR Model - please supply model=<Class>" unless args[:class]
+    raise "ERROR: Could not find file #{args[:input]}" unless File.exists?(args[:input])
+
+    klass =  Kernal.const_get(args[:model])
+    raise "ERROR: No such AR Model found - please check model=<Class>" unless(klass)
+
+    require 'product_loader'
+    require 'method_mapper_excel'
+
+    args[:class]
+
+    @method_mapper = MethodMapperExcel.new(args[:input], Product)
+
+    @excel = @method_mapper.excel
+
+    if(args[:verbose])
+      puts "Loading from Excel file: #{args[:input]}"
+      puts "Processing #{@excel.num_rows} rows"
+    end
+
+    # TODO create YAML configuration file to drive mandatory columns
+    #
+    # TODO create YAML configuration file to drive defaults etc
+  
+    # Process spreadsheet and create model instances
+    
+    method_names = @method_mapper.method_names
+
+    Product.transaction do
+      @products =  []
+
+      (1..@excel.num_rows).collect do |row|
+
+        product_data_row = @excel.sheet.getRow(row)
+        break if product_data_row.nil?
+
+        # Excel num_rows seems to return all 'visible' rows so,
+        # we have to manually detect when actual data ends and all the empty rows start
+        contains_data = required_methods.find { |mthd| ! product_data_row.getCell(method_names.index(mthd)).to_s.empty? }
+        break unless contains_data
+ 
+        @assoc_classes = {}
+
+        loader = ProductLoader.new()
+
+        # TODO - Smart sorting of column processing order ....
+        # Does not currently ensure mandatory columns (for valid?) processed first but model needs saving
+        # before associations can be processed so user should ensure mandatory columns are prior to associations
+
+        @method_mapper.methods.each_with_index do |method_map, col|
+
+          loader.process(method_map, @excel.value(product_data_row, col))
+          begin
+            loader.load_object.save if( loader.load_object.valid? && loader.load_object.new_record? )
+          rescue
+            raise "Error processing Product"
+          end
+        end
+
+        product = loader.load_object
+
+        product.available_on ||= Time.now.to_s(:db)
+
+        # TODO - handle when it's not valid ? 
+        # Process rest and dump out an exception list of Products
+        #unless(product.valid?)
+        #end
+
+        puts "SAVING ROW #{row} : #{product.inspect}" if args[:verbose]
+
+        unless(product.save)
+          puts product.errors.inspect
+          puts product.errors.full_messages.inspect
+          raise "Error Saving Product: #{product.sku} :#{product.name}"
+        else
+          @products << product
+        end
+      end
+    end   # TRANSACTION
+
+  end
+
+end

data/tasks/file_tasks.rake

@@ -0,0 +1,38 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Feb 2011
+# License::   MIT. Free, Open Source.
+#
+# Usage::     rake autotelik:file_rename input=/blah image_load input=path_to_images
+#
+
+namespace :autotelik do
+
+  desc "copy or mv a folder of files, consistently renaming in the process"
+  task :file_rename, :input, :offset, :prefix, :width, :commit, :mv do |t, args|
+    raise "USAGE: rake file_rename input='C:\blah' [offset=n prefix='str' width=n]" unless args[:input] && File.exists?(args[:input])
+    width = args[:width] || 2
+
+    action = args[:mv] ? 'mv' : 'cp'
+
+    cache = args[:input]
+
+    if(File.exists?(cache) )
+      puts "Renaming files from #{cache}"
+      Dir.glob(File.join(cache, "*")) do |name|
+        path, base_name = File.split(name)
+        id = base_name.slice!(/\w+/)
+
+        id = id.to_i + args[:offset].to_i if(args[:offset])
+        id = "%0#{width}d" % id.to_i if(args[:width])
+        id = args[:prefix] + id.to_s if(args[:prefix])
+
+        destination = File.join(path, "#{id}#{base_name}")
+        puts "ACTION: #{action} #{name} #{destination}"
+
+        File.send( action, name, destination) if args[:commit]
+      end
+    end
+  end
+
+end

data/tasks/seed_fu_product_template.erb

@@ -0,0 +1,15 @@
+p = Product.seed( :name ) do |s|
+  s.name =  <%= @name %>
+  s.available_on =  '2009-09-01 09:00:00.0'
+  s.meta_keywords =  ['training', 'training']
+  s.meta_description =  ""
+  s.description =  '<%= @description %>'
+  s.price = 0.00
+  s.sku = '<%= @sku %>'
+
+  s.is_physical    = false
+  s.is_private     = false
+
+  s.append_association :taxons, Taxon.find_by_name( 'Training' )
+
+end