ar_loader 0.0.4 → 0.0.6

data/tasks/excel_loader.rake

@@ -1,101 +1,114 @@
-# Copyright:: (c) Autotelik Media Ltd 2011
-# Author ::   Tom Statter
-# Date ::     Feb 2011
-# License::   TBD. Free, Open Source. MIT ?
-#
-# REQUIRES:   JRuby
-#
-# Usage from rake : jruby -S rake excel_loader input=<file.xls>
-#
-# e.g.  => jruby -S rake excel_load input=vendor\extensions\autotelik\fixtures\ExampleInfoWeb.xls
-#       => jruby -S rake excel_load input=C:\MyProducts.xls verbose=true
-#
-namespace :autotelik do
-
-  desc "Populate AR model's table with data stored in Excel"
-  task :excel_load, :klass, :input, :verbose, :sku_prefix, :needs => :environment do |t, args|
-
-    raise "USAGE: jruby -S rake excel_load input=excel_file.xls" unless args[:input]
-    raise "ERROR: Cannot process without AR Model - please supply model=<Class>" unless args[:class]
-    raise "ERROR: Could not find file #{args[:input]}" unless File.exists?(args[:input])
-
-    klass =  Kernal.const_get(args[:model])
-    raise "ERROR: No such AR Model found - please check model=<Class>" unless(klass)
-
-    require 'product_loader'
-    require 'method_mapper_excel'
-
-    args[:class]
-
-    @method_mapper = MethodMapperExcel.new(args[:input], Product)
-
-    @excel = @method_mapper.excel
-
-    if(args[:verbose])
-      puts "Loading from Excel file: #{args[:input]}"
-      puts "Processing #{@excel.num_rows} rows"
-    end
-
-    # TODO create YAML configuration file to drive mandatory columns
-    #
-    # TODO create YAML configuration file to drive defaults etc
-  
-    # Process spreadsheet and create model instances
-    
-    method_names = @method_mapper.method_names
-
-    Product.transaction do
-      @products =  []
-
-      (1..@excel.num_rows).collect do |row|
-
-        product_data_row = @excel.sheet.getRow(row)
-        break if product_data_row.nil?
-
-        # Excel num_rows seems to return all 'visible' rows so,
-        # we have to manually detect when actual data ends and all the empty rows start
-        contains_data = required_methods.find { |mthd| ! product_data_row.getCell(method_names.index(mthd)).to_s.empty? }
-        break unless contains_data
- 
-        @assoc_classes = {}
-
-        loader = ProductLoader.new()
-
-        # TODO - Smart sorting of column processing order ....
-        # Does not currently ensure mandatory columns (for valid?) processed first but model needs saving
-        # before associations can be processed so user should ensure mandatory columns are prior to associations
-
-        @method_mapper.methods.each_with_index do |method_map, col|
-
-          loader.process(method_map, @excel.value(product_data_row, col))
-          begin
-            loader.load_object.save if( loader.load_object.valid? && loader.load_object.new_record? )
-          rescue
-            raise "Error processing Product"
-          end
-        end
-
-        product = loader.load_object
-
-        product.available_on ||= Time.now.to_s(:db)
-
-        # TODO - handle when it's not valid ? 
-        # Process rest and dump out an exception list of Products
-        #unless(product.valid?)
-        #end
-
-        puts "SAVING ROW #{row} : #{product.inspect}" if args[:verbose]
-
-        unless(product.save)
-          puts product.errors.inspect
-          puts product.errors.full_messages.inspect
-          raise "Error Saving Product: #{product.sku} :#{product.name}"
-        else
-          @products << product
-        end
-      end
-    end   # TRANSACTION
-
-  end
-
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Feb 2011
+# License::   TBD. Free, Open Source. MIT ?
+#
+# REQUIRES:   JRuby
+#
+# Usage::
+# => jruby -S rake autotelik:excel_load model=<active record class> input=<file.xls>
+# => jruby -S rake autotelik:excel_load model=<active record class> input=C:\MyProducts.xlsverbose=true
+#
+namespace :autotelik do
+
+  desc "Populate a model's table in db with data from .xls (Excel) file"
+  task :excel_load, :model, :loader, :input, :verbose, :needs => :environment do |t, args|
+
+    raise "USAGE: jruby -S rake excel_load input=excel_file.xls" unless args[:input]
+    raise "ERROR: Cannot process without AR Model - please supply model=<Class>" unless args[:model]
+    raise "ERROR: Could not find file #{args[:input]}" unless File.exists?(args[:input])
+
+    begin
+      klass = Kernel.const_get(args[:model])
+    rescue NameError
+      raise "ERROR: No such AR Model found - please check model=<Class>"
+    end
+
+    begin
+      require "#{args[:model]}_loader"
+
+      loader_klass = Kernel.const_get("#{args[:model]}Loader")
+
+      loader = loader_klass.new(klass)
+    rescue
+      puts "INFO: No specific #{args[:model]}Loader found using generic loader"
+      loader = LoaderBase.new(klass)
+    end
+
+    require 'method_mapper_excel'
+
+    @method_mapper = MethodMapperExcel.new(args[:input], klass)
+
+    @excel = @method_mapper.excel
+
+    if(args[:verbose])
+      puts "Loading from Excel file: #{args[:input]}"
+      puts "Processing #{@excel.num_rows} rows"
+    end
+
+    # Process spreadsheet and create model instances
+ 
+    klass.transaction do
+      @loaded_objects =  []
+
+      (1..@excel.num_rows).collect do |row|
+
+        data_row = @excel.sheet.getRow(row)
+        break if data_row.nil?
+
+        @assoc_classes = {}
+
+        # TODO - Smart sorting of column processing order ....
+        # Does not currently ensure mandatory columns (for valid?) processed first but model needs saving
+        # before associations can be processed so user should ensure mandatory columns are prior to associations
+
+        contains_data = false
+
+        # Iterate over the columns method_mapper found in Excel,
+        # pulling data out of associated column
+        @method_mapper.methods.each_with_index do |method_map, col|
+
+          value = @excel.value(data_row, col)
+
+          # Excel num_rows seems to return all 'visible' rows so,
+          # we have to manually detect when actual data ends, this isn't very smart but
+          # currently got no better idea than ending once we hit the first completely empty row
+ 
+          contains_data = true if(value.to_s.empty?)
+
+          puts "VALUE #{value.class}"
+          puts "VALUE #{value} #{value.inspect}"
+
+          loader.process(method_map, @excel.value(data_row, col))
+
+          begin
+            loader.load_object.save if( loader.load_object.valid? && loader.load_object.new_record? )
+          rescue
+            raise "Error processing row"
+          end
+        end
+
+        break unless contains_data
+
+        loaded_object = loader.load_object
+
+        # TODO - handle when it's not valid ? 
+        # Process rest and dump out an exception list of Products
+        #unless(product.valid?)
+        #end
+
+        puts "SAVING ROW #{row} : #{loaded_object.inspect}" if args[:verbose]
+
+        unless(loaded_object.save)
+          puts loaded_object.errors.inspect
+          puts loaded_object.errors.full_messages.inspect
+          raise "Error Saving : #{loaded_object.inspect}"
+        else
+          @loaded_objects << loaded_object
+        end
+      end
+    end   # TRANSACTION
+
+  end
+
 end

data/tasks/file_tasks.rake

@@ -1,38 +1,38 @@
-# Copyright:: (c) Autotelik Media Ltd 2011
-# Author ::   Tom Statter
-# Date ::     Feb 2011
-# License::   MIT. Free, Open Source.
-#
-# Usage::     rake autotelik:file_rename input=/blah image_load input=path_to_images
-#
-
-namespace :autotelik do
-
-  desc "copy or mv a folder of files, consistently renaming in the process"
-  task :file_rename, :input, :offset, :prefix, :width, :commit, :mv do |t, args|
-    raise "USAGE: rake file_rename input='C:\blah' [offset=n prefix='str' width=n]" unless args[:input] && File.exists?(args[:input])
-    width = args[:width] || 2
-
-    action = args[:mv] ? 'mv' : 'cp'
-
-    cache = args[:input]
-
-    if(File.exists?(cache) )
-      puts "Renaming files from #{cache}"
-      Dir.glob(File.join(cache, "*")) do |name|
-        path, base_name = File.split(name)
-        id = base_name.slice!(/\w+/)
-
-        id = id.to_i + args[:offset].to_i if(args[:offset])
-        id = "%0#{width}d" % id.to_i if(args[:width])
-        id = args[:prefix] + id.to_s if(args[:prefix])
-
-        destination = File.join(path, "#{id}#{base_name}")
-        puts "ACTION: #{action} #{name} #{destination}"
-
-        File.send( action, name, destination) if args[:commit]
-      end
-    end
-  end
-
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Feb 2011
+# License::   MIT. Free, Open Source.
+#
+# Usage::     rake autotelik:file_rename input=/blah image_load input=path_to_images
+#
+
+namespace :autotelik do
+
+  desc "copy or mv a folder of files, consistently renaming in the process"
+  task :file_rename, :input, :offset, :prefix, :width, :commit, :mv do |t, args|
+    raise "USAGE: rake file_rename input='C:\blah' [offset=n prefix='str' width=n]" unless args[:input] && File.exists?(args[:input])
+    width = args[:width] || 2
+
+    action = args[:mv] ? 'mv' : 'cp'
+
+    cache = args[:input]
+
+    if(File.exists?(cache) )
+      puts "Renaming files from #{cache}"
+      Dir.glob(File.join(cache, "*")) do |name|
+        path, base_name = File.split(name)
+        id = base_name.slice!(/\w+/)
+
+        id = id.to_i + args[:offset].to_i if(args[:offset])
+        id = "%0#{width}d" % id.to_i if(args[:width])
+        id = args[:prefix] + id.to_s if(args[:prefix])
+
+        destination = File.join(path, "#{id}#{base_name}")
+        puts "ACTION: #{action} #{name} #{destination}"
+
+        File.send( action, name, destination) if args[:commit]
+      end
+    end
+  end
+
 end

data/tasks/spree/image_load.rake

@@ -1,103 +1,108 @@
-# Copyright:: (c) Autotelik Media Ltd 2011
-# Author ::   Tom Statter
-# Date ::     Feb 2011
-# License::   MIT. Free, Open Source.
-#
-# Usage from rake : rake image_load input=path_to_images
-#
-# => rake image_load input=vendor\extensions\autotelik\fixtures\
-# => rake image_load input="C:\images\photos large' dummy=true
-# => rake image_load input="C:\images\taxon_icons" skip_if_no_assoc=true klass=Taxon
-
-namespace :autotelik do
-
-  namespace :spree do
-
-    desc "Populate the DB with images.\nDefault location db/image_seeds, or specify :input=<path> or dir under db/image_seeds with :folder"
-    # :dummy => dummy run without actual saving to DB
-    task :image_load, :input, :folder, :dummy, :sku, :skip_if_no_assoc, :skip_if_loaded, :klass, :needs => :environment do |t, args|
-
-      require 'image_loader'
-
-      raise "USAGE: Please specify one of :input or :folder" if(args[:input] && args[:folder])
-      puts  "SKU not specified " if(args[:input] && args[:folder])
-
-      if args[:input]
-        @image_cache = args[:input]
-      else
-        @image_cache =  File.join(Rails.root, "db", "image_seeds")
-        @image_cache =  File.join(@image_cache, args[:folder]) if(args[:folder])
-      end
-
-      klazz = args[:klass] ? Kernal.const_get(args[:klass]) : Product
-
-      image_loader = ImageLoader.new
-
-      if(File.exists? @image_cache )
-        puts "Loading images from #{@image_cache}"
-
-        missing_records = []
-        Dir.glob("#{@image_cache}/*.{jpg,png,gif}") do |image_name|
-
-          puts "Processing #{image_name} : #{File.exists?(image_name)}"
-          base_name = File.basename(image_name, '.*')
-
-          record = nil
-          if(klazz == Product && args[:sku])
-            sku = base_name.slice!(/\w+/)
-            sku.strip!
-            base_name.strip!
-
-            puts "Looking fo SKU #{sku}"
-            record = Variant.find_by_sku(sku)
-            if record
-              record = record.product   # SKU stored on Variant but we want it's master Product
-            else
-              puts "Looking for NAME [#{base_name}]"
-              record = klazz.find_by_name(base_name)
-            end
-          else
-            puts "Looking for #{klazz.name} with NAME [#{base_name}]"
-            record = klazz.find_by_name(base_name)
-          end
-      
-          if(record)
-            puts "FOUND: #{record.inspect}"
-            puts "FOUND: #{record.images.collect(&:attachment_file_name).inspect}"
-            exists = record.images.detect {|i| puts "COMPARE #{i.attachment_file_name} => #{image_name}"; i.attachment_file_name == image_name }
-            puts "Check for existing attachment [#{exists}]"
-            if(args[:skip_if_loaded] && exists)
-              puts "Skipping - Image #{image_name} already loaded for #{klazz}"
-              next
-            end
-          else
-            missing_records << image_name
-          end
-
-          # Now do actual upload to DB unless we are doing a dummy run,
-          # or the Image must have an associated record
-          unless(args[:dummy] == 'true' || (args[:skip_if_no_assoc] && record.nil?))
-            image_loader.refresh()
-            puts "Process Image"
-            image_loader.process( image_name, record )
-          end
-
-        end
-
-        unless missing_records.empty?
-          FileUtils.mkdir_p('MissingRecords') unless File.directory?('MissingRecords')
-        
-          puts '\nMISSING Records Report>>'
-          missing_records.each do |i|
-            puts "Copy #{i} to MissingRecords folder"
-            FileUtils.cp( i, 'MissingRecords')  unless(args[:dummy] == 'true')
-          end
-        end
-
-      else
-        puts "ERROR: Supplied Path #{@image_cache} not accesible"
-        exit(-1)
-      end
-    end
-  end
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Feb 2011
+# License::   MIT. Free, Open Source.
+#
+# Usage from rake : rake image_load input=path_to_images
+#
+# => rake image_load input=vendor\extensions\autotelik\fixtures\
+# => rake image_load input="C:\images\photos large' dummy=true
+# => rake image_load input="C:\images\taxon_icons" skip_if_no_assoc=true klass=Taxon
+
+namespace :autotelik do
+
+  namespace :spree do
+
+    desc "Populate the DB with images.\nDefault location db/image_seeds, or specify :input=<path> or dir under db/image_seeds with :folder"
+    # :dummy => dummy run without actual saving to DB
+    task :image_load, :input, :folder, :dummy, :sku, :skip_if_no_assoc, :skip_if_loaded, :model, :needs => :environment do |t, args|
+
+      require 'image_loader'
+
+      raise "USAGE: Please specify one of :input or :folder" if(args[:input] && args[:folder])
+      puts  "SKU not specified " if(args[:input] && args[:folder])
+
+      if args[:input]
+        @image_cache = args[:input]
+      else
+        @image_cache =  File.join(Rails.root, "db", "image_seeds")
+        @image_cache =  File.join(@image_cache, args[:folder]) if(args[:folder])
+      end
+
+      attachment_klazz = Product
+
+      begin
+        attachment_klazz = Kernel.const_get(args[:model]) if(args[:model])
+      rescue NameError
+        attachment_klazz = Product
+      end
+
+      image_loader = ImageLoader.new
+
+      if(File.exists? @image_cache )
+        puts "Loading images from #{@image_cache}"
+
+        missing_records = []
+        Dir.glob("#{@image_cache}/*.{jpg,png,gif}") do |image_name|
+
+          puts "Processing #{image_name} : #{File.exists?(image_name)}"
+          base_name = File.basename(image_name, '.*')
+
+          record = nil
+          if(attachment_klazz == Product && args[:sku])
+            sku = base_name.slice!(/\w+/)
+            sku.strip!
+            base_name.strip!
+
+            puts "Looking fo SKU #{sku}"
+            record = Variant.find_by_sku(sku)
+            if record
+              record = record.product   # SKU stored on Variant but we want it's master Product
+            else
+              puts "Looking for NAME [#{base_name}]"
+              record = attachment_klazz.find_by_name(base_name)
+            end
+          else
+            puts "Looking for #{attachment_klazz.name} with NAME [#{base_name}]"
+            record = attachment_klazz.find_by_name(base_name)
+          end
+      
+          if(record)
+            puts "Found record for attachment : #{record.inspect}"
+            exists = record.images.detect {|i| puts "COMPARE #{i.attachment_file_name} => #{image_name}"; i.attachment_file_name == image_name }
+            puts "Found existing attachments [#{exists}]" unless(exists.nil?)
+            if(args[:skip_if_loaded] && !exists.nil?)
+              puts "Skipping - Image #{image_name} already loaded for #{attachment_klazz}"
+              next
+            end
+          else
+            missing_records << image_name
+          end
+
+          # Now do actual upload to DB unless we are doing a dummy run,
+          # or the Image must have an associated record
+          unless(args[:dummy] == 'true' || (args[:skip_if_no_assoc] && record.nil?))
+            image_loader.reset()
+            puts "Process Image"
+            image_loader.process( image_name, record )
+          end
+
+        end
+
+        unless missing_records.empty?
+          FileUtils.mkdir_p('MissingRecords') unless File.directory?('MissingRecords')
+        
+          puts '\nMISSING Records Report>>'
+          missing_records.each do |i|
+            puts "Copy #{i} to MissingRecords folder"
+            FileUtils.cp( i, 'MissingRecords')  unless(args[:dummy] == 'true')
+          end
+        end
+
+      else
+        puts "ERROR: Supplied Path #{@image_cache} not accesible"
+        exit(-1)
+      end
+    end
+  end
 end