ar_loader 0.0.6 → 0.0.8

data/tasks/config/seed_fu_product_template.erb

@@ -1,15 +1,15 @@
-p = Product.seed( :name ) do |s|
-  s.name =  <%= @name %>
-  s.available_on =  '2009-09-01 09:00:00.0'
-  s.meta_keywords =  ['training', 'training']
-  s.meta_description =  ""
-  s.description =  '<%= @description %>'
-  s.price = 0.00
-  s.sku = '<%= @sku %>'
-
-  s.is_physical    = false
-  s.is_private     = false
-
-  s.append_association :taxons, Taxon.find_by_name( 'Training' )
-
-end
+p = Product.seed( :name ) do |s|
+  s.name =  <%= @name %>
+  s.available_on =  '2009-09-01 09:00:00.0'
+  s.meta_keywords =  ['training', 'training']
+  s.meta_description =  ""
+  s.description =  '<%= @description %>'
+  s.price = 0.00
+  s.sku = '<%= @sku %>'
+
+  s.is_physical    = false
+  s.is_private     = false
+
+  s.append_association :taxons, Taxon.find_by_name( 'Training' )
+
+end

data/tasks/config/tidy_config.txt

@@ -1,13 +1,13 @@
-# Copyright:: (c) Autotelik Media Ltd 2011
-# Author ::   Tom Statter
-# Date ::     Aug 2010
-#
-# License::   Free, OpenSource... MIT ?
-
-# This is the best effort I've found so far to reduce the amount of MS cruft
-# to absolute minimum
-# ... but unfortunately these tags will NOT BE REMOVED completely - manual cleanup still required
-
-# TODO - add another ruby parse layer to strip these out completely
-
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2010
+#
+# License::   Free, OpenSource... MIT ?
+
+# This is the best effort I've found so far to reduce the amount of MS cruft
+# to absolute minimum
+# ... but unfortunately these tags will NOT BE REMOVED completely - manual cleanup still required
+
+# TODO - add another ruby parse layer to strip these out completely
+
 new-empty-tags: o:smarttagtype, st1:placename, st1:place, st1:placetype

data/tasks/db_tasks.rake

@@ -1,65 +1,65 @@
-# Author ::   Tom Statter
-# Date ::     Mar 2011
-#
-# License::   The MIT License (Free and OpenSource)
-#
-# About::     Additional Rake tasks useful when testing seeding DB via ARLoader
-#
-namespace :autotelik do
-
-  namespace :db  do
-
-    SYSTEM_TABLE_EXCLUSION_LIST = ['schema_migrations']
-
-    desc "Purge the current database"
-    task :purge, :exclude_system_tables, :needs => [:environment] do |t, args|
-      require 'highline/import'
-
-      if(RAILS_ENV == 'production')
-        agree("WARNING: In Production database, REALLY PURGE ? [y]:")
-      end
-
-      config = ActiveRecord::Base.configurations[RAILS_ENV || 'development']
-      case config['adapter']
-      when "mysql", "jdbcmysql"
-        ActiveRecord::Base.establish_connection(config)
-        ActiveRecord::Base.connection.tables.each do |table|
-          next if(args[:exclude_system_tables] && SYSTEM_TABLE_EXCLUSION_LIST.include?(table) )
-          puts "purging table: #{table}"
-          ActiveRecord::Base.connection.execute("TRUNCATE #{table}")
-        end
-      when "sqlite","sqlite3"
-        dbfile = config["database"] || config["dbfile"]
-        File.delete(dbfile) if File.exist?(dbfile)
-      when "sqlserver"
-        dropfkscript = "#{config["host"]}.#{config["database"]}.DP1".gsub(/\\/,'-')
-        `osql -E -S #{config["host"]} -d #{config["database"]} -i db\\#{dropfkscript}`
-        `osql -E -S #{config["host"]} -d #{config["database"]} -i db\\#{RAILS_ENV}_structure.sql`
-      when "oci", "oracle"
-        ActiveRecord::Base.establish_connection(config)
-        ActiveRecord::Base.connection.structure_drop.split(";\n\n").each do |ddl|
-          ActiveRecord::Base.connection.execute(ddl)
-        end
-      when "firebird"
-        ActiveRecord::Base.establish_connection(config)
-        ActiveRecord::Base.connection.recreate_database!
-      else
-        raise "Task not supported by '#{config["adapter"]}'"
-      end
-    end
- 
-    desc "Clear database and optional directories such as assets, then run db:seed"
-    task :seed_again, :assets, :needs => [:environment] do |t, args|
-
-      Rake::Task['autotelik:db:purge'].invoke( true )   # i.e ENV['exclude_system_tables'] = true
-
-      if(args[:assets])
-        assets = "#{Rails.root}/public/assets"
-        FileUtils::rm_rf(assets) if(File.exists?(assets))
-      end
-
-      Rake::Task['db:seed'].invoke
-    end
-  
-  end # db
+# Author ::   Tom Statter
+# Date ::     Mar 2011
+#
+# License::   The MIT License (Free and OpenSource)
+#
+# About::     Additional Rake tasks useful when testing seeding DB via ARLoader
+#
+namespace :autotelik do
+
+  namespace :db  do
+
+    SYSTEM_TABLE_EXCLUSION_LIST = ['schema_migrations']
+
+    desc "Purge the current database"
+    task :purge, :exclude_system_tables, :needs => [:environment] do |t, args|
+      require 'highline/import'
+
+      if(RAILS_ENV == 'production')
+        agree("WARNING: In Production database, REALLY PURGE ? [y]:")
+      end
+
+      config = ActiveRecord::Base.configurations[RAILS_ENV || 'development']
+      case config['adapter']
+      when "mysql", "jdbcmysql"
+        ActiveRecord::Base.establish_connection(config)
+        ActiveRecord::Base.connection.tables.each do |table|
+          next if(args[:exclude_system_tables] && SYSTEM_TABLE_EXCLUSION_LIST.include?(table) )
+          puts "purging table: #{table}"
+          ActiveRecord::Base.connection.execute("TRUNCATE #{table}")
+        end
+      when "sqlite","sqlite3"
+        dbfile = config["database"] || config["dbfile"]
+        File.delete(dbfile) if File.exist?(dbfile)
+      when "sqlserver"
+        dropfkscript = "#{config["host"]}.#{config["database"]}.DP1".gsub(/\\/,'-')
+        `osql -E -S #{config["host"]} -d #{config["database"]} -i db\\#{dropfkscript}`
+        `osql -E -S #{config["host"]} -d #{config["database"]} -i db\\#{RAILS_ENV}_structure.sql`
+      when "oci", "oracle"
+        ActiveRecord::Base.establish_connection(config)
+        ActiveRecord::Base.connection.structure_drop.split(";\n\n").each do |ddl|
+          ActiveRecord::Base.connection.execute(ddl)
+        end
+      when "firebird"
+        ActiveRecord::Base.establish_connection(config)
+        ActiveRecord::Base.connection.recreate_database!
+      else
+        raise "Task not supported by '#{config["adapter"]}'"
+      end
+    end
+ 
+    desc "Clear database and optional directories such as assets, then run db:seed"
+    task :seed_again, :assets, :needs => [:environment] do |t, args|
+
+      Rake::Task['autotelik:db:purge'].invoke( true )   # i.e ENV['exclude_system_tables'] = true
+
+      if(args[:assets])
+        assets = "#{Rails.root}/public/assets"
+        FileUtils::rm_rf(assets) if(File.exists?(assets))
+      end
+
+      Rake::Task['db:seed'].invoke
+    end
+  
+  end # db
 end # autotelik

data/tasks/excel_loader.rake

@@ -1,114 +1,64 @@
-# Copyright:: (c) Autotelik Media Ltd 2011
-# Author ::   Tom Statter
-# Date ::     Feb 2011
-# License::   TBD. Free, Open Source. MIT ?
-#
-# REQUIRES:   JRuby
-#
-# Usage::
-# => jruby -S rake autotelik:excel_load model=<active record class> input=<file.xls>
-# => jruby -S rake autotelik:excel_load model=<active record class> input=C:\MyProducts.xlsverbose=true
-#
-namespace :autotelik do
-
-  desc "Populate a model's table in db with data from .xls (Excel) file"
-  task :excel_load, :model, :loader, :input, :verbose, :needs => :environment do |t, args|
-
-    raise "USAGE: jruby -S rake excel_load input=excel_file.xls" unless args[:input]
-    raise "ERROR: Cannot process without AR Model - please supply model=<Class>" unless args[:model]
-    raise "ERROR: Could not find file #{args[:input]}" unless File.exists?(args[:input])
-
-    begin
-      klass = Kernel.const_get(args[:model])
-    rescue NameError
-      raise "ERROR: No such AR Model found - please check model=<Class>"
-    end
-
-    begin
-      require "#{args[:model]}_loader"
-
-      loader_klass = Kernel.const_get("#{args[:model]}Loader")
-
-      loader = loader_klass.new(klass)
-    rescue
-      puts "INFO: No specific #{args[:model]}Loader found using generic loader"
-      loader = LoaderBase.new(klass)
-    end
-
-    require 'method_mapper_excel'
-
-    @method_mapper = MethodMapperExcel.new(args[:input], klass)
-
-    @excel = @method_mapper.excel
-
-    if(args[:verbose])
-      puts "Loading from Excel file: #{args[:input]}"
-      puts "Processing #{@excel.num_rows} rows"
-    end
-
-    # Process spreadsheet and create model instances
- 
-    klass.transaction do
-      @loaded_objects =  []
-
-      (1..@excel.num_rows).collect do |row|
-
-        data_row = @excel.sheet.getRow(row)
-        break if data_row.nil?
-
-        @assoc_classes = {}
-
-        # TODO - Smart sorting of column processing order ....
-        # Does not currently ensure mandatory columns (for valid?) processed first but model needs saving
-        # before associations can be processed so user should ensure mandatory columns are prior to associations
-
-        contains_data = false
-
-        # Iterate over the columns method_mapper found in Excel,
-        # pulling data out of associated column
-        @method_mapper.methods.each_with_index do |method_map, col|
-
-          value = @excel.value(data_row, col)
-
-          # Excel num_rows seems to return all 'visible' rows so,
-          # we have to manually detect when actual data ends, this isn't very smart but
-          # currently got no better idea than ending once we hit the first completely empty row
- 
-          contains_data = true if(value.to_s.empty?)
-
-          puts "VALUE #{value.class}"
-          puts "VALUE #{value} #{value.inspect}"
-
-          loader.process(method_map, @excel.value(data_row, col))
-
-          begin
-            loader.load_object.save if( loader.load_object.valid? && loader.load_object.new_record? )
-          rescue
-            raise "Error processing row"
-          end
-        end
-
-        break unless contains_data
-
-        loaded_object = loader.load_object
-
-        # TODO - handle when it's not valid ? 
-        # Process rest and dump out an exception list of Products
-        #unless(product.valid?)
-        #end
-
-        puts "SAVING ROW #{row} : #{loaded_object.inspect}" if args[:verbose]
-
-        unless(loaded_object.save)
-          puts loaded_object.errors.inspect
-          puts loaded_object.errors.full_messages.inspect
-          raise "Error Saving : #{loaded_object.inspect}"
-        else
-          @loaded_objects << loaded_object
-        end
-      end
-    end   # TRANSACTION
-
-  end
-
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Feb 2011
+# License::   TBD. Free, Open Source. MIT ?
+#
+# REQUIRES:   JRuby
+#
+# Usage::
+#
+#  In Rakefile:
+#
+#     require 'ar_loader'
+#
+#     ArLoader::load_tasks
+#
+#  Cmd Line:
+#
+# => jruby -S rake ar_loader:excel model=<active record class> input=<file.xls>
+# => jruby -S rake ar_loader:excel model=<active record class> input=C:\MyProducts.xlsverbose=true
+#
+require 'ar_loader'
+require 'excel_loader'
+
+namespace :ar_loader do
+
+  desc "Populate a model's table in db with data from .xls (Excel) file"
+  
+  task :excel, [:model, :loader, :input, :verbose] => [:environment] do |t, args|
+
+    # in familiar ruby style args seems to have been become empty using this new style for rake 0.9.2
+    #  whatever format i try, on both Win and OSX .. so had to revert back to ENV
+    model = ENV['model']
+    input = ENV['input']
+    
+    raise "USAGE: jruby -S rake ar_loader:excel input=excel_file.xls model=<Class>" unless(input)
+    raise "ERROR: Cannot process without AR Model - please supply model=<Class>" unless(model)
+    raise "ERROR: Could not find file #{args[:input]}" unless File.exists?(input)
+
+    begin
+      klass = Kernel.const_get(model)
+    rescue NameError
+      raise "ERROR: No such AR Model found - check valid model supplied via model=<Class>"
+    end
+
+    if(ENV['loader'])
+      begin
+        loader_klass = Kernel.const_get(ENV['loader'])
+
+        loader = loader_klass.new(klass)
+
+        puts "INFO: Using loader : #{loader.class}"
+      rescue
+        puts "INFO: No specific #{model}Loader found  - using generic ExcelLoader"
+        loader = ARLoader::ExcelLoader.new(klass)
+      end
+    else
+      puts "INFO: No Loader specified - using generic ExcelLoader"
+      loader = ARLoader::ExcelLoader.new(klass)
+    end
+
+    loader.load(input)
+  end
+
 end

data/tasks/file_tasks.rake

@@ -1,38 +1,37 @@
-# Copyright:: (c) Autotelik Media Ltd 2011
-# Author ::   Tom Statter
-# Date ::     Feb 2011
-# License::   MIT. Free, Open Source.
-#
-# Usage::     rake autotelik:file_rename input=/blah image_load input=path_to_images
-#
-
-namespace :autotelik do
-
-  desc "copy or mv a folder of files, consistently renaming in the process"
-  task :file_rename, :input, :offset, :prefix, :width, :commit, :mv do |t, args|
-    raise "USAGE: rake file_rename input='C:\blah' [offset=n prefix='str' width=n]" unless args[:input] && File.exists?(args[:input])
-    width = args[:width] || 2
-
-    action = args[:mv] ? 'mv' : 'cp'
-
-    cache = args[:input]
-
-    if(File.exists?(cache) )
-      puts "Renaming files from #{cache}"
-      Dir.glob(File.join(cache, "*")) do |name|
-        path, base_name = File.split(name)
-        id = base_name.slice!(/\w+/)
-
-        id = id.to_i + args[:offset].to_i if(args[:offset])
-        id = "%0#{width}d" % id.to_i if(args[:width])
-        id = args[:prefix] + id.to_s if(args[:prefix])
-
-        destination = File.join(path, "#{id}#{base_name}")
-        puts "ACTION: #{action} #{name} #{destination}"
-
-        File.send( action, name, destination) if args[:commit]
-      end
-    end
-  end
-
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Feb 2011
+# License::   MIT
+#
+# Usage::     rake ar_loader:file_rename input=/blah image_load input=path_to_images
+#
+namespace :ar_loader do
+
+  desc "copy or mv a folder of files, consistently renaming in the process"
+  task :file_rename, :input, :offset, :prefix, :width, :commit, :mv do |t, args|
+    raise "USAGE: rake file_rename input='C:\blah' [offset=n prefix='str' width=n]" unless args[:input] && File.exists?(args[:input])
+    width = args[:width] || 2
+
+    action = args[:mv] ? 'mv' : 'cp'
+
+    cache = args[:input]
+
+    if(File.exists?(cache) )
+      puts "Renaming files from #{cache}"
+      Dir.glob(File.join(cache, "*")) do |name|
+        path, base_name = File.split(name)
+        id = base_name.slice!(/\w+/)
+
+        id = id.to_i + args[:offset].to_i if(args[:offset])
+        id = "%0#{width}d" % id.to_i if(args[:width])
+        id = args[:prefix] + id.to_s if(args[:prefix])
+
+        destination = File.join(path, "#{id}#{base_name}")
+        puts "ACTION: #{action} #{name} #{destination}"
+
+        File.send( action, name, destination) if args[:commit]
+      end
+    end
+  end
+
 end

data/tasks/loader.rake

@@ -0,0 +1,45 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2011
+# License::   MIT
+#
+# Usage::
+#
+#  In Rakefile:
+#
+#     require 'ar_loader'
+#
+#     ArLoader::load_tasks
+#
+#  Cmd Line:
+#
+# => jruby -S rake ar_loader:csv model=<active record class> input=<file.csv>
+#
+namespace :ar_loader do
+
+  desc "Populate a model's table in db with data from CSV file"
+  task :csv, [:model, :loader, :input, :verbose] => [:environment] do |t, args|
+
+    # in familiar ruby style args seems to have been become empty with rake 0.9.2 whatever i try
+    # so had to revert back to ENV
+    model = ENV['model']
+    input = ENV['input']
+   
+    raise "USAGE: jruby -S rake ar_loader:excel input=excel_file.xls model=<Class>" unless(input)
+    raise "ERROR: Cannot process without AR Model - please supply model=<Class>" unless(model)
+    raise "ERROR: Could not find file #{args[:input]}" unless File.exists?(input)
+
+    begin
+      klass = Kernel.const_get(model)
+    rescue NameError
+      raise "ERROR: No such AR Model found - check valid model supplied via model=<Class>"
+    end
+
+    puts "INFO: Using CSV loader"
+    
+    loader = CsvLoader.new(klass)
+
+    loader.load(input)
+  end
+
+end