ar_loader 0.0.4

data/tasks/spree/image_load.rake

@@ -0,0 +1,103 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Feb 2011
+# License::   MIT. Free, Open Source.
+#
+# Usage from rake : rake image_load input=path_to_images
+#
+# => rake image_load input=vendor\extensions\autotelik\fixtures\
+# => rake image_load input="C:\images\photos large' dummy=true
+# => rake image_load input="C:\images\taxon_icons" skip_if_no_assoc=true klass=Taxon
+
+namespace :autotelik do
+
+  namespace :spree do
+
+    desc "Populate the DB with images.\nDefault location db/image_seeds, or specify :input=<path> or dir under db/image_seeds with :folder"
+    # :dummy => dummy run without actual saving to DB
+    task :image_load, :input, :folder, :dummy, :sku, :skip_if_no_assoc, :skip_if_loaded, :klass, :needs => :environment do |t, args|
+
+      require 'image_loader'
+
+      raise "USAGE: Please specify one of :input or :folder" if(args[:input] && args[:folder])
+      puts  "SKU not specified " if(args[:input] && args[:folder])
+
+      if args[:input]
+        @image_cache = args[:input]
+      else
+        @image_cache =  File.join(Rails.root, "db", "image_seeds")
+        @image_cache =  File.join(@image_cache, args[:folder]) if(args[:folder])
+      end
+
+      klazz = args[:klass] ? Kernal.const_get(args[:klass]) : Product
+
+      image_loader = ImageLoader.new
+
+      if(File.exists? @image_cache )
+        puts "Loading images from #{@image_cache}"
+
+        missing_records = []
+        Dir.glob("#{@image_cache}/*.{jpg,png,gif}") do |image_name|
+
+          puts "Processing #{image_name} : #{File.exists?(image_name)}"
+          base_name = File.basename(image_name, '.*')
+
+          record = nil
+          if(klazz == Product && args[:sku])
+            sku = base_name.slice!(/\w+/)
+            sku.strip!
+            base_name.strip!
+
+            puts "Looking fo SKU #{sku}"
+            record = Variant.find_by_sku(sku)
+            if record
+              record = record.product   # SKU stored on Variant but we want it's master Product
+            else
+              puts "Looking for NAME [#{base_name}]"
+              record = klazz.find_by_name(base_name)
+            end
+          else
+            puts "Looking for #{klazz.name} with NAME [#{base_name}]"
+            record = klazz.find_by_name(base_name)
+          end
+      
+          if(record)
+            puts "FOUND: #{record.inspect}"
+            puts "FOUND: #{record.images.collect(&:attachment_file_name).inspect}"
+            exists = record.images.detect {|i| puts "COMPARE #{i.attachment_file_name} => #{image_name}"; i.attachment_file_name == image_name }
+            puts "Check for existing attachment [#{exists}]"
+            if(args[:skip_if_loaded] && exists)
+              puts "Skipping - Image #{image_name} already loaded for #{klazz}"
+              next
+            end
+          else
+            missing_records << image_name
+          end
+
+          # Now do actual upload to DB unless we are doing a dummy run,
+          # or the Image must have an associated record
+          unless(args[:dummy] == 'true' || (args[:skip_if_no_assoc] && record.nil?))
+            image_loader.refresh()
+            puts "Process Image"
+            image_loader.process( image_name, record )
+          end
+
+        end
+
+        unless missing_records.empty?
+          FileUtils.mkdir_p('MissingRecords') unless File.directory?('MissingRecords')
+        
+          puts '\nMISSING Records Report>>'
+          missing_records.each do |i|
+            puts "Copy #{i} to MissingRecords folder"
+            FileUtils.cp( i, 'MissingRecords')  unless(args[:dummy] == 'true')
+          end
+        end
+
+      else
+        puts "ERROR: Supplied Path #{@image_cache} not accesible"
+        exit(-1)
+      end
+    end
+  end
+end

data/tasks/spree/product_loader.rake

@@ -0,0 +1,107 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Feb 2011
+# License::   MIT. Free, Open Source.
+#
+# REQUIRES:   JRuby access to Java
+#
+# Usage from rake : jruby -S rake excel_loader input=<file.xls>
+#
+# e.g.  => jruby -S rake excel_load input=vendor\extensions\autotelik\fixtures\ExampleInfoWeb.xls
+#       => jruby -S rake excel_load input=C:\MyProducts.xls verbose=true
+#
+namespace :autotelik do
+
+  desc "Populate a Spree database with Product/Varient data stored in Excel"
+  task :product_load, :input, :verbose, :sku_prefix, :needs => :environment do |t, args|
+
+    raise "USAGE: jruby -S rake product_load input=excel_file.xls" unless args[:input]
+    raise "ERROR: Could not find file #{args[:input]}" unless File.exists?(args[:input])
+
+    require 'product_loader'
+    require 'method_mapper_excel'
+
+    @method_mapper = MethodMapperExcel.new(args[:input], Product)
+
+    @excel = @method_mapper.excel
+
+    if(args[:verbose])
+      puts "Loading from Excel file: #{args[:input]}"
+      puts "Processing #{@excel.num_rows} rows"
+    end
+
+    # REQUIRED 'set' methods on Product i.e will not validate/save without these
+    required_methods = ['sku', 'name', 'price']
+
+    @method_mapper.check_mandatory( required_methods )
+
+    # COLUMNS WITH DEFAULTS - TODO create YAML configuration file to drive defaults etc
+  
+    MethodDetail.set_default_value('available_on', Time.now.to_s(:db) )
+    MethodDetail.set_default_value('cost_price', 0.0 )
+
+    MethodDetail.set_prefix('sku', args[:sku_prefix] ) if args[:sku_prefix]
+
+    # Process spreadsheet and create Products
+    method_names = @method_mapper.method_names
+
+    sku_index = method_names.index('sku')
+
+    Product.transaction do
+      @products =  []
+
+      (1..@excel.num_rows).collect do |row|
+
+        product_data_row = @excel.sheet.getRow(row)
+        break if product_data_row.nil?
+
+        # Excel num_rows seems to return all 'visible' rows so,
+        # we have to manually detect when actual data ends and all the empty rows start
+        contains_data = required_methods.find { |mthd| ! product_data_row.getCell(method_names.index(mthd)).to_s.empty? }
+        break unless contains_data
+ 
+        @assoc_classes = {}
+
+        loader = ProductLoader.new()
+
+        # TODO - Smart sorting of column processing order ....
+        # Does not currently ensure mandatory columns (for valid?) processed first but Product needs saving
+        # before associations can be processed so user should ensure SKU, name, price columns are among first columns
+
+        @method_mapper.methods.each_with_index do |method_map, col|
+          product_data_row.getCell(col).setCellType(JExcelFile::HSSFCell::CELL_TYPE_STRING) if(col == sku_index)
+          loader.process(method_map, @excel.value(product_data_row, col))
+          begin
+            prod = loader.load_object
+            if( prod.valid? && prod.new_record? )
+              prod.save
+            end
+          rescue
+            raise "Error processing Product"
+          end
+        end
+
+        product = loader.load_object
+
+        product.available_on ||= Time.now.to_s(:db)
+
+        # TODO - handle when it's not valid ? 
+        # Process rest and dump out an exception list of Products
+        #unless(product.valid?)
+        #end
+
+        puts "SAVING ROW #{row} : #{product.inspect}" if args[:verbose]
+
+        unless(product.save)
+          puts product.errors.inspect
+          puts product.errors.full_messages.inspect
+          raise "Error Saving Product: #{product.sku} :#{product.name}"
+        else
+          @products << product
+        end
+      end
+    end   # TRANSACTION
+
+  end
+
+end

data/tasks/tidy_config.txt

@@ -0,0 +1,13 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2010
+#
+# License::   Free, OpenSource... MIT ?
+
+# This is the best effort I've found so far to reduce the amount of MS cruft
+# to absolute minimum
+# ... but unfortunately these tags will NOT BE REMOVED completely - manual cleanup still required
+
+# TODO - add another ruby parse layer to strip these out completely
+
+new-empty-tags: o:smarttagtype, st1:placename, st1:place, st1:placetype

data/tasks/word_to_seedfu.rake

@@ -0,0 +1,167 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2010
+#
+# License::   Free, OpenSource... MIT ?
+#
+# About::     Rake tasks to read Word documents, containing product descriptions,
+#             convert to HTML, tidy the HTML and then create seed_fu ready fixtures,
+#             from a template, with product description supplied by the HTML
+#
+#             Note cleanest HTML is produced by this combination : saving with WdFormatHTML
+#             not WdFormatFilteredHTML and using the '--word-2000', 'y' option to tidy
+#             (don't use the '--bare' option)
+#
+#             Not currently available for JRuby due to Win32Ole requirement
+#
+#             Requires local exes available in PATH for :
+#             Microsoft Word
+#             HTML Tidy - http://tidy.sourceforge.net  (Free)
+#
+require 'erb'
+
+namespace :autotelik do
+
+  desc "Convert MS Word to HTML and seed_fu fixtures. help=true for detailed usage."
+
+  task :word2html, :help, :needs => [:environment] do |t, args|
+    x =<<-EOS
+
+  USAGE::
+      Convert MS Word docs to HTML and seed_fu fixtures, by default searches for docs
+      in RAILS_ROOT/doc/copy
+
+      You can change the directory where Word document files are located
+      with the COPY_PATH  environment variable.
+
+      Examples:
+        # default, to convert all Word files for the current environment
+        rake autotelik:word2seedfu
+
+        # to load seed files matching orders or customers
+        rake db:seed SEED=orders,customers
+
+        # to load files from RAILS_ROOT/features/fixtures
+        rake db:seed FIXTURE_PATH=features/fixtures
+    EOS
+
+    if(args[:help])
+      puts x
+      exit(0)
+    end
+
+    site_extension_lib = File.join(SiteExtension.root, 'lib')
+
+    require File.join(site_extension_lib, 'word')
+
+    copy_path = ENV["COPY_PATH"] ? ENV["COPY_PATH"] : File.join(RAILS_ROOT, "doc", "copy")
+    fixtures_path = ENV["FIXTURES_PATH"] ? ENV["FIXTURES_PATH"] : File.join(RAILS_ROOT, "db", "fixtures")
+
+    copy_files = Dir[File.join(copy_path, '*.doc')]
+
+    copy_files.each do |file|
+
+      name = File.basename(file, '.doc')
+
+      puts "\n== Generate raw HTML from #{name}.doc =="
+
+      @word = Word.new(true)
+
+      @word.open( file )
+
+      html_file = File.join(copy_path, "#{name}.ms.html")
+
+      @word.save_as_html( html_file )
+
+      tidy_file = File.join(copy_path, "#{name}.html")
+
+      tidy_config = File.join(site_extension_lib, 'tasks', 'tidy_config.txt')
+
+      puts "tidy cmd line:", "tidy -config #{tidy_config} -clean --show-body-only y --word-2000 y --indent-spaces 2 -output #{tidy_file} #{html_file}"
+
+      result = system("tidy", '-config', "#{tidy_config}", '-clean', '--show-body-only', 'y', '--word-2000', 'y', '--indent-spaces', '2', '-output', "#{tidy_file}", "#{html_file}")
+
+      # TODO maybe report on result, $?
+
+      File.open( tidy_file ) do |f|
+        puts f.read
+      end
+
+      @word.quit
+    end
+  end
+
+  desc "Convert MS Word to HTML and seed_fu fixtures. help=true for detailed usage."
+  task :word2seedfu => :environment do
+    site_extension_lib = File.join(SiteExtension.root, 'lib')
+
+    require File.join(site_extension_lib, 'word')
+
+    sku_id     = ENV["INITIAL_SKU_ID"] ? ENV["INITIAL_SKU_ID"] : 0
+    sku_prefix = ENV["SKU_PREFIX"] ? ENV["SKU_PREFIX"] : File.basename( RAILS_ROOT )
+
+    seedfu_template = File.join(site_extension_lib, 'tasks', 'seed_fu_product_template.erb')
+
+    begin
+      File.open( seedfu_template ) do |f|
+        @template = ERB.new(f.read)
+      end
+    rescue => e
+      puts "ERROR: #{e.inspect}"
+      puts "Cannot open or read template #{seedfu_template}"
+      raise e
+    end
+
+    copy_path = ENV["COPY_PATH"] ? ENV["COPY_PATH"] : File.join(RAILS_ROOT, "doc", "copy")
+    fixtures_path = ENV["FIXTURES_PATH"] ? ENV["FIXTURES_PATH"] : File.join(RAILS_ROOT, "db", "fixtures")
+
+    copy_files = Dir[File.join(copy_path, '*.doc')]
+
+    copy_files.each do |file|
+
+      name = File.basename(file, '.doc')
+
+      puts "\n== Generate raw HTML from #{name}.doc =="
+
+      @word = Word.new(true)
+
+      @word.open( file )
+
+      html_file = File.join(copy_path, "#{name}.ms.html")
+
+      @word.save_as_html( html_file )
+
+      tidy_file = File.join(copy_path, "#{name}.html")
+
+      tidy_config = File.join(site_extension_lib, 'tasks', 'tidy_config.txt')
+
+      puts "tidy cmd line:", "tidy -config #{tidy_config} -clean --show-body-only y --word-2000 y --indent-spaces 2 -output #{tidy_file} #{html_file}"
+
+      result = system("tidy", '-config', "#{tidy_config}", '-clean', '--show-body-only', 'y', '--word-2000', 'y', '--indent-spaces', '2', '-output', "#{tidy_file}", "#{html_file}")
+
+      # TODO maybe report on result, $?
+
+      File.open( tidy_file ) do |f|
+        @description = f.read
+      end
+
+      sku_id_str  = "%03d" % sku_id
+
+      seed_file = "#{sku_id_str}_#{name.gsub(' ', '_')}.rb"
+      puts "\n== Generate seed fu file #{seed_file} =="
+
+      @sku  = "#{sku_prefix}_#{sku_id_str}"
+      @name = 'TODO'
+  
+      File.open( File.join(fixtures_path, seed_file), 'w' ) do |f|
+        f.write @template.result(binding)
+        puts "\nFile created: #{File.join(fixtures_path, seed_file)}"
+      end
+
+      sku_id += 1
+
+      @word.quit
+    end
+
+  end
+end

metadata

@@ -0,0 +1,90 @@
+--- !ruby/object:Gem::Specification 
+name: ar_loader
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.0.4
+platform: ruby
+authors: 
+  - thomas statter
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-03-24 00:00:00 +00:00
+default_executable: 
+dependencies: []
+
+description: A file based loader for Active Record models. Seed database directly from Excel/CSV. Includes rake support for Spree
+email: rubygems@autotelik.co.uk
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+  - README.markdown
+  - LICENSE
+files: 
+  - LICENSE
+  - README.markdown
+  - Rakefile
+  - lib/ar_loader.rb
+  - lib/to_b.rb
+  - lib/VERSION
+  - lib/engine/file_definitions.rb
+  - lib/engine/mapping_file_definitions.rb
+  - lib/engine/method_detail.rb
+  - lib/engine/method_mapper.rb
+  - lib/engine/method_mapper_csv.rb
+  - lib/engine/word.rb
+  - lib/engine/jruby/jexcel_file.rb
+  - lib/engine/jruby/method_mapper_excel.rb
+  - lib/java/poi-3.2-FINAL-20081019.jar
+  - lib/java/poi-3.6.jar
+  - lib/java/poi-contrib-3.6-20091214.jar
+  - lib/java/poi-examples-3.6-20091214.jar
+  - lib/java/poi-ooxml-3.6-20091214.jar
+  - lib/java/poi-ooxml-schemas-3.6-20091214.jar
+  - lib/java/poi-scratchpad-3.6-20091214.jar
+  - lib/loaders/loader_base.rb
+  - lib/loaders/spree/image_loader.rb
+  - lib/loaders/spree/product_loader.rb
+  - spec/excel_loader_spec.rb
+  - spec/spec_helper.rb
+  - tasks/db_tasks.rake
+  - tasks/excel_loader.rake
+  - tasks/file_tasks.rake
+  - tasks/seed_fu_product_template.erb
+  - tasks/tidy_config.txt
+  - tasks/word_to_seedfu.rake
+  - tasks/spree/image_load.rake
+  - tasks/spree/product_loader.rake
+has_rdoc: true
+homepage: http://www.autotelik.co.uk
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+  - lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.5.1
+signing_key: 
+specification_version: 3
+summary: File based loader for Active Record models
+test_files: []
+