datashift 0.2.2 → 0.4.0

data/README.markdown

@@ -2,6 +2,8 @@
 
 Provides tools to shift data between Ruby, external applications, files and ActiveRecord.
 
+Wiki taking shape with more info here : <b>https://github.com/autotelik/datashift/wiki</b>
+
 ### Features
 
 Map Active Record models and associations to files, generate sample templates.
@@ -39,17 +41,23 @@ To pull the tasks in, add this call to your Rakefile :
 
      DataShift::load_tasks
 
+To keep the availability to only development mode use
+
+    if(Rails.env.development?)
+        DataShift::<b>load_tasks</b>
+    end
+
 To use the Thor command line applications :
 
     Create a high level .thor file - e.g mysite.thor - in your applications root directory 
 
 Pull in the thor commands :
-
+```
     require 'thor'
     require 'datashift'
 
-    DataShift::load_commands
-
+    DataShift::<b>load_commands</b>
+```
 To check the available tasks run
 
     bundle exec rake -T datashift
@@ -58,6 +66,10 @@ and/or
 
     bundle exc thor list datashift
 
+To get usgae information use thor help <command>, for example
+
+    bundle exec thor help datashift:generate:excel
+
 N.B - To use the Excel loader, OLE and Excel are NOT required, however
 JRuby is required, since it uses Java's Apache POI under the hood to process .xls files.
 

data/VERSION

@@ -1 +1 @@
-0.2.2
+0.4.0

data/datashift.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "datashift"
-  s.version = "0.2.2"
+  s.version = "0.4.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Thomas Statter"]
-  s.date = "2012-03-05"
+  s.date = "2012-03-12"
   s.description = "A suite of tools to move data between ActiveRecord models,databases,applications like Excel/Open Office, files and projects including Spree"
   s.email = "rubygems@autotelik.co.uk"
   s.extra_rdoc_files = [
@@ -30,11 +30,14 @@ Gem::Specification.new do |s|
     "lib/datashift.rb",
     "lib/datashift/exceptions.rb",
     "lib/datashift/file_definitions.rb",
+    "lib/datashift/logging.rb",
     "lib/datashift/mapping_file_definitions.rb",
     "lib/datashift/method_detail.rb",
+    "lib/datashift/method_details_manager.rb",
     "lib/datashift/method_dictionary.rb",
     "lib/datashift/method_mapper.rb",
     "lib/datashift/model_mapper.rb",
+    "lib/datashift/populator.rb",
     "lib/exporters/csv_exporter.rb",
     "lib/exporters/excel_exporter.rb",
     "lib/exporters/exporter_base.rb",
@@ -66,6 +69,9 @@ Gem::Specification.new do |s|
     "lib/loaders/spree/image_loader.rb",
     "lib/loaders/spree/product_loader.rb",
     "lib/thor/generate_excel.thor",
+    "lib/thor/import_excel.thor",
+    "lib/thor/spree/bootstrap_cleanup.thor",
+    "lib/thor/spree/products_images.thor",
     "public/spree/products/large/DEMO_001_ror_bag.jpeg",
     "public/spree/products/large/DEMO_002_Powerstation.jpg",
     "public/spree/products/large/DEMO_003_ror_mug.jpeg",
@@ -116,6 +122,8 @@ Gem::Specification.new do |s|
     "spec/fixtures/simple_template_spec.xls",
     "spec/fixtures/spree/SpreeProducts.csv",
     "spec/fixtures/spree/SpreeProducts.xls",
+    "spec/fixtures/spree/SpreeProductsDefaults.yml",
+    "spec/fixtures/spree/SpreeProductsMandatoryOnly.xls",
     "spec/fixtures/spree/SpreeProductsMultiColumn.csv",
     "spec/fixtures/spree/SpreeProductsMultiColumn.xls",
     "spec/fixtures/spree/SpreeProductsSimple.csv",
@@ -131,6 +139,7 @@ Gem::Specification.new do |s|
     "spec/spec_helper.rb",
     "spec/spree_exporter_spec.rb",
     "spec/spree_generator_spec.rb",
+    "spec/spree_images_loader_spec.rb",
     "spec/spree_loader_spec.rb",
     "spec/spree_method_mapping_spec.rb",
     "tasks/config/seed_fu_product_template.erb",
@@ -140,7 +149,6 @@ Gem::Specification.new do |s|
     "tasks/import/csv.rake",
     "tasks/import/excel.rake",
     "tasks/spree/image_load.rake",
-    "tasks/spree/product_loader.rake",
     "tasks/word_to_seedfu.rake",
     "test/helper.rb",
     "test/test_interact.rb"

data/lib/applications/jruby/jexcel_file.rb

@@ -272,10 +272,17 @@ if(DataShift::Guards::jruby?)
     end
     
     def save( filename = nil )
+   
       filename.nil? ? file = @filepath : file = filename
-      out = FileOutputStream.new(file)
-      @book.write(out) unless @book.nil?
-      out.close
+      begin
+        out = FileOutputStream.new(file)
+        @book.write(out) unless @book.nil?
+
+        out.close
+      rescue => e
+        puts e
+        raise "Cannot write file - is file already open in Excel ?"
+      end
     end
 
     def save_to_text( filename )

data/lib/datashift.rb

@@ -1,11 +1,32 @@
-# Copyright:: (c) Autotelik Media Ltd 2011
+# Copyright:: (c) Autotelik Media Ltd 2010 - 2012 Tom Statter
 # Author ::   Tom Statter
 # Date ::     Aug 2010
-# License::   TBD. Free, Open Source. MIT ?
+# License::   Free, Open Source.
 #
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+
 # Details::   Active Record Loader
 # 
-# #  To pull Datashift commands into your main application :
+# To pull DataShift commands into your main application :
 #
 #     require 'datashift'
 #
@@ -93,6 +114,7 @@ module DataShift
     require_libs.each do |base|
       Dir[File.join(library_path, base, '*.rb')].each do |rb|
         unless File.directory? rb
+          #puts rb
           require rb
         end
       end
@@ -112,72 +134,13 @@ module DataShift
   # Load all the datashift Thor commands and make them available throughout app
 
   def self.load_commands()
-    
     base = File.join(library_path, 'thor', '**')
     
     Dir["#{base}/*.thor"].each do |f|
-    
       next unless File.file?(f)
       Thor::Util.load_thorfile(f)
     end
   end
-
-
-  def self.load_thorfiles(dir)
-    Dir.chdir(dir) do
-      thor_files = Dir.glob('**/*.thor').delete_if { |x| not File.file?(x) }
-      thor_files.each do |f|
-        Thor::Util.load_thorfile(f)
-      end
-    end
-  end
-
-
-  module Logging
-    
-    class MultiIO
-           
-      def initialize(*targets)
-        @targets = []
-        targets.each {|t| @targets << Logger.new(t) }
-      end
-
-      def add(target)
-        @targets << Logger.new(target)
-      end
-      
-      
-      def method_missing(method, *args, &block)
-        @targets.each {|t| t.send(method, *args, &block) }
-      end
-    
-      def verbose
-        add(STDOUT)
-      end
-    
-    end
-    
-    require 'logger'
-     
-    def logdir
-      @logdir ||= 'log'
-      @logdir
-    end
-  
-    def logger
-      @logger ||= open
-      @logger
-    end
-    
-    private
-    
-    def open( log = 'datashift.log')
-      FileUtils::mkdir(logdir) unless File.directory?(logdir)
-      log_file = File.open( File.join(logdir(), 'datashift.log'), "a")
-      @logger = MultiIO.new(log_file)
-      @logger
-    end
-  end
   
 end
 

data/lib/datashift/exceptions.rb

@@ -3,6 +3,7 @@ module DataShift
   class BadRuby < StandardError; end
   
   class UnsupportedFileType < StandardError; end
+  class BadFile < StandardError; end
   
   class MappingDefinitionError < StandardError; end
 

data/lib/datashift/logging.rb

@@ -0,0 +1,58 @@
+# Copyright:: (c) Autotelik Media Ltd 2012
+# Author ::   Tom Statter
+# Date ::     Aug 2012
+# License::   Free, Open Source. MIT.
+#
+# Details::   Logging facilities for datashift.
+#
+require 'fileutils'
+  
+module DataShift
+
+  module Logging
+    
+    class MultiIO
+           
+      def initialize(*targets)
+        @targets = []
+        targets.each {|t| @targets << Logger.new(t) }
+      end
+
+      def add(target)
+        @targets << Logger.new(target)
+      end
+      
+      
+      def method_missing(method, *args, &block)
+        @targets.each {|t| t.send(method, *args, &block) }
+      end
+    
+      def verbose
+        add(STDOUT)
+      end
+    
+    end
+    
+    require 'logger'
+     
+    def logdir
+      @logdir ||= 'log'
+      @logdir
+    end
+  
+    def logger
+      @logger ||= open
+      @logger
+    end
+    
+    private
+    
+    def open( log = 'datashift.log')
+      return ActiveRecord::Base.logger if(defined?(ActiveRecord) && ActiveRecord::Base.logger)
+      FileUtils::mkdir(logdir) unless File.directory?(logdir)
+      log_file = File.open( File.join(logdir(), 'datashift.log'), "a")
+      MultiIO.new(log_file)
+    end
+  end
+  
+end

data/lib/datashift/method_detail.rb

@@ -14,49 +14,10 @@ require 'to_b'
 
 module DataShift
 
-  # Stores MethodDetails for a class mapped by type
-  class MethodDetailsManager
-
-    attr_reader :method_details
-    
-    def initialize( klass )
-      @parent_class = klass
-      @method_details = {}
-      @method_details_list = {}
-    end
-    
-    def add(method_details)
-      @method_details[method_details.operator_type] ||= {}
-      @method_details_list[method_details.operator_type] ||= []
-       
-      @method_details[method_details.operator_type][method_details.name] = method_details
-      @method_details_list[method_details.operator_type] << method_details
-      @method_details_list[method_details.operator_type].uniq!
-    end
-
-    def <<(method_details)
-      add(method_details)      
-    end
-
-    def find(name, type)
-      method_details = get(type)
-     
-      method_details ?  method_details[name] : nil
-    end
-    
-    # type is expected to be one of MethodDetail::supportedtype_enum
-    def get( type )
-      @method_details[type]
-    end
-    
-    def get_list( type )
-      @method_details_list[type]
-    end
-    
-  end
-  
   class MethodDetail
 
+    include DataShift::Logging
+    
     def self.supported_types_enum
       @type_enum ||= Set[:assignment, :belongs_to, :has_one, :has_many]
       @type_enum
@@ -149,7 +110,7 @@ module DataShift
 
       elsif( operator_for(:has_many) )
         
-        puts "DEBUG : VALUE TYPE [#{value.class.name.include?(operator.classify)}] [#{ModelMapper.class_from_string(value.class.name)}]" unless(value.is_a?(Array))
+        #puts "DEBUG : VALUE TYPE [#{value.class.name.include?(operator.classify)}] [#{ModelMapper.class_from_string(value.class.name)}]" unless(value.is_a?(Array))
      
         # The include? check is best I can come up with right now .. to handle module/namespaces
         # TODO - can we determine the real class type of an association
@@ -168,20 +129,20 @@ module DataShift
         if(value.is_a?(operator_class))
           record.send(operator + '=', value)
         else
-          puts "ERROR #{value.class} - Not expected type for has_one #{operator} - cannot assign"
+          logger.error("ERROR #{value.class} - Not expected type for has_one #{operator} - cannot assign")
           # TODO -  Not expected type - maybe try to look it up somehow ?"
           #insistent_has_many(record, @current_value)
         end
 
       elsif( operator_for(:assignment) && @col_type )
         #puts "DEBUG : COl TYPE defined for #{@name} : #{@assignment} => #{@current_value} #{@col_type.type}"
-        #puts "DEBUG : COl TYPE CAST: #{@current_value} => #{@col_type.type_cast( @current_value ).inspect}"
+       # puts "DEBUG : Column [#{@name}] : COl TYPE CAST: #{@current_value} => #{@col_type.type_cast( @current_value ).inspect}"
         record.send( operator + '=' , @col_type.type_cast( @current_value ) )
 
         #puts "DEBUG : MethodDetails Assignment RESULT: #{record.send(operator)}"
 
       elsif( operator_for(:assignment) )
-        #puts "DEBUG : Brute force assignment of value  #{@current_value} supplied for Column [#{@name}]"
+        #puts "DEBUG : Column [#{@name}] : Brute force assignment of value  #{@current_value}"
         # brute force case for assignments without a column type (which enables us to do correct type_cast)
         # so in this case, attempt straightforward assignment then if that fails, basic ops such as to_s, to_i, to_f etc
         insistent_assignment(record, @current_value)

data/lib/datashift/method_details_manager.rb

@@ -0,0 +1,54 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2010
+# License::   MIT
+#
+# Details::   This class provides info and access to groups of accessor methods,
+#             grouped by AR model. 
+#
+require 'to_b'
+
+module DataShift
+
+  # Stores MethodDetails for a class mapped by type
+  class MethodDetailsManager
+
+    attr_reader :method_details
+    
+    def initialize( klass )
+      @parent_class = klass
+      @method_details = {}
+      @method_details_list = {}
+    end
+    
+    def add(method_details)
+      @method_details[method_details.operator_type.to_sym] ||= {}
+      @method_details_list[method_details.operator_type.to_sym] ||= []
+       
+      @method_details[method_details.operator_type.to_sym][method_details.name] = method_details
+      @method_details_list[method_details.operator_type.to_sym] << method_details
+      @method_details_list[method_details.operator_type.to_sym].uniq!
+    end
+
+    def <<(method_details)
+      add(method_details)      
+    end
+
+    def find(name, type)
+      method_details = get(type)
+     
+      method_details ?  method_details[name] : nil
+    end
+    
+    # type is expected to be one of MethodDetail::supportedtype_enum
+    def get( type )
+      @method_details[type.to_sym]
+    end
+    
+    def get_list( type )
+      @method_details_list[type.to_sym]
+    end
+    
+  end
+  
+end