datashift 0.8.0 → 0.9.0

data/README.markdown

@@ -43,11 +43,23 @@ To use :
 
 To pull the tasks in, add this call to your Rakefile :
 
+<<<<<<< HEAD
+```ruby 
+    DataShift::load_tasks
+```
+
+To keep the availability to only development mode use
+
+```ruby 
+    DataShift::load_tasks if(Rails.env.development?)
+```
+=======
 ```ruby DataShift::load_tasks```
 
 To keep the availability to only development mode use
 
 ```ruby DataShift::load_tasks if(Rails.env.development?)```
+>>>>>>> a6d5d492fd9df38a4acb8c50423219994b4b5601
 
 To use the Thor command line applications :
 

data/Rakefile

@@ -34,13 +34,15 @@ Jeweler::Tasks.new do |gem|
   gem.homepage = "http://github.com/autotelik/datashift"
   gem.license = "MIT"
   gem.summary = %Q{ Shift data betwen applications and Active Record}
-  gem.description = %Q{A suite of tools to move data between ActiveRecord models,databases,applications like Excel/Open Office, files and projects including Spree}
+  gem.description = %Q{Comprehensive Excel and CSV import/export tools. Shift data between ActiveRecord databases, applications, and projects like Spree}
   gem.email = "rubygems@autotelik.co.uk"
   gem.authors = ["Thomas Statter"]
   # dependencies defined in Gemfile
   gem.files.exclude ['sandbox']
   
-  gem.add_dependency ['spreadsheet']
+  gem.add_dependency 'spreadsheet'
+  gem.add_dependency 'rubyzip'
+  
 end
 Jeweler::RubygemsDotOrgTasks.new
 

data/VERSION

@@ -1 +1 @@
-0.8.0
+0.9.0

data/datashift.gemspec

@@ -5,12 +5,12 @@
 
 Gem::Specification.new do |s|
   s.name = "datashift"
-  s.version = "0.8.0"
+  s.version = "0.9.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Thomas Statter"]
-  s.date = "2012-07-09"
-  s.description = "A suite of tools to move data between ActiveRecord models,databases,applications like Excel/Open Office, files and projects including Spree"
+  s.date = "2012-09-03"
+  s.description = "Comprehensive Excel and CSV import/export tools. Shift data between ActiveRecord databases, applications, and projects like Spree"
   s.email = "rubygems@autotelik.co.uk"
   s.extra_rdoc_files = [
     "LICENSE.txt",
@@ -28,6 +28,7 @@ Gem::Specification.new do |s|
     "lib/applications/jruby/jexcel_file.rb",
     "lib/applications/jruby/word.rb",
     "lib/datashift.rb",
+    "lib/datashift/delimiters.rb",
     "lib/datashift/exceptions.rb",
     "lib/datashift/file_definitions.rb",
     "lib/datashift/logging.rb",
@@ -69,12 +70,13 @@ Gem::Specification.new do |s|
     "lib/loaders/spreadsheet_loader.rb",
     "lib/loaders/spree/image_loader.rb",
     "lib/loaders/spree/product_loader.rb",
-    "lib/thor/export_excel.thor",
+    "lib/thor/export.thor",
     "lib/thor/generate_excel.thor",
-    "lib/thor/import_excel.thor",
+    "lib/thor/import.thor",
     "lib/thor/spree/bootstrap_cleanup.thor",
     "lib/thor/spree/products_images.thor",
     "lib/thor/spree/reports.thor",
+    "lib/thor/tools.thor",
     "public/spree/products/large/DEMO_001_ror_bag.jpeg",
     "public/spree/products/large/DEMO_002_Powerstation.jpg",
     "public/spree/products/large/DEMO_003_ror_mug.jpeg",
@@ -90,12 +92,8 @@ Gem::Specification.new do |s|
     "public/spree/products/small/DEMO_001_ror_bag.jpeg",
     "public/spree/products/small/DEMO_002_Powerstation.jpg",
     "public/spree/products/small/DEMO_003_ror_mug.jpeg",
-    "sandbox/app/controllers/application_controller.rb",
-    "sandbox/config/application.rb",
-    "sandbox/config/database.yml",
-    "sandbox/config/environment.rb",
-    "sandbox/config/environments/development.rb",
     "spec/Gemfile",
+    "spec/csv_exporter_spec.rb",
     "spec/csv_loader_spec.rb",
     "spec/datashift_spec.rb",
     "spec/db/migrate/20110803201325_create_test_bed.rb",
@@ -147,13 +145,13 @@ Gem::Specification.new do |s|
     "spec/spree_images_loader_spec.rb",
     "spec/spree_loader_spec.rb",
     "spec/spree_method_mapping_spec.rb",
+    "spec/spree_variants_loader_spec.rb",
     "spec/thor_spec.rb",
     "tasks/config/seed_fu_product_template.erb",
     "tasks/config/tidy_config.txt",
     "tasks/db_tasks.rake",
     "tasks/export/excel_generator.rake",
     "tasks/file_tasks.rake",
-    "tasks/import/csv.rake",
     "tasks/import/excel.rake",
     "tasks/word_to_seedfu.rake",
     "test/helper.rb",
@@ -162,7 +160,7 @@ Gem::Specification.new do |s|
   s.homepage = "http://github.com/autotelik/datashift"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.15"
+  s.rubygems_version = "1.8.24"
   s.summary = "Shift data betwen applications and Active Record"
 
   if s.respond_to? :specification_version then
@@ -170,11 +168,14 @@ Gem::Specification.new do |s|
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<spreadsheet>, [">= 0"])
+      s.add_runtime_dependency(%q<rubyzip>, [">= 0"])
     else
       s.add_dependency(%q<spreadsheet>, [">= 0"])
+      s.add_dependency(%q<rubyzip>, [">= 0"])
     end
   else
     s.add_dependency(%q<spreadsheet>, [">= 0"])
+    s.add_dependency(%q<rubyzip>, [">= 0"])
   end
 end
 

data/lib/datashift/delimiters.rb

@@ -0,0 +1,87 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2010
+# License::   MIT
+#
+#  Details::  Module providing standard location for delimiters used in both export/import
+#  
+#             For example we support multiple entries in a single column, so the string 
+#             needs to be formatted with recognisable delimiters seperating each of the multiple values.
+#
+module DataShift
+
+ 
+  module Delimiters
+
+
+    # Support multiple associations being added to a base object to be specified in a single column.
+    # 
+    # Entry represents the association to find via supplied name, value to use in the lookup.
+    # Can contain multiple lookup name/value pairs, separated by multi_assoc_delim ( | )
+    # 
+    # Default syntax :
+    #
+    #   Name1:value1, value2|Name2:value1, value2, value3|Name3:value1, value2
+    #
+    # E.G.
+    #   Association Properties, has a column named Size, and another called Colour,
+    #   and this combination could be used to lookup multiple associations to add to the main model Jumper
+    #
+    #       Size:small            # => generates find_by_size( 'small' )
+    #       Size:large            # => generates find_by_size( 'large' )
+    #       Colour:red,green,blue # => generates find_all_by_colour( ['red','green','blue'] )
+    #
+    #       Size:large|Size:medium|Size:large
+    #         => Find 3 different associations, perform lookup via column called Size
+    #         => Jumper.properties << [ small, medium, large ]
+    #
+    def self.name_value_delim
+      @name_value_delim ||= ':'
+      @name_value_delim
+    end
+
+    def self.set_name_value_delim(x)  @name_value_delim = x; end
+    # TODO - support embedded object creation/update via hash (which hopefully we should be able to just forward to AR)
+    #
+    #      |Category|
+    #      name:new{ :date => '20110102', :owner = > 'blah'}
+    #
+    
+    
+    def self.multi_value_delim
+      @multi_value_delim ||= ','
+      @multi_value_delim
+    end
+    
+    def self.set_multi_value_delim(x) @multi_value_delim = x; end
+    
+    # Multiple objects can be embedded in single columns.
+    # In this example a single Category column contains 3 separate entries, New, SecondHand, Retro
+    # object creation/update via hash (which hopefully we should be able to just forward to AR)
+    #
+    #      | Category |
+    #      'name =>New, :a => 1, :b => 2|name => SecondHand, :a => 6, :b => 34|Name:Old, :a => 12, :b => 67', 'Next Column'
+    #
+    def self.multi_assoc_delim
+      @multi_assoc_delim ||= '|'
+      @multi_assoc_delim
+    end
+
+
+    def self.set_multi_assoc_delim(x) @multi_assoc_delim = x; end
+    
+    
+    def self.csv_delim
+      @csv_delim ||= ','
+      @csv_delim
+    end
+    
+    def self.set_csv_delim(x) @csv_delim = x; end
+    
+    def self.eol
+      "\n"
+    end
+
+  end
+
+end

data/lib/datashift/method_detail.rb

@@ -25,6 +25,11 @@ module DataShift
       @type_enum
     end
 
+    def self.association_types_enum
+      @assoc_type_enum ||= Set[:belongs_to, :has_one, :has_many]
+      @assoc_type_enum
+    end
+    
     # When looking up an association, try each of these in turn till a match
     #  i.e find_by_name .. find_by_title and so on, lastly try the raw id
     @@insistent_find_by_list ||= [:name, :title, :id]

data/lib/datashift/method_details_manager.rb

@@ -47,9 +47,13 @@ module DataShift
     end
     
     def get_list( type )
-      @method_details_list[type.to_sym]
+      @method_details_list[type.to_sym] || []
     end
     
+    def get_operators( op_type )
+      get_list(op_type).collect { |md| md.operator }
+    end
+     
     def all_available_operators
       method_details_list.values.flatten.collect(&:operator)
     end

data/lib/datashift/method_dictionary.rb

@@ -59,7 +59,9 @@ module DataShift
       # the raw form i.e without the '='  for consistency 
       if( options[:reload] || assignments[klass].nil? )
  
-        assignments[klass] = klass.column_names
+        # TODO investigate difference with attribute_names - maybe column names can be assigned to an attribute
+        # so in terms of method calls on klass attribute_names might be safer
+        assignments[klass] = klass.column_names  
            
         if(options[:instance_methods] == true)
           setters = klass.instance_methods.grep(/\w+=/).collect {|x| x.to_s }

data/lib/exporters/csv_exporter.rb

@@ -7,30 +7,178 @@
 #
 #
 require 'exporter_base'
+require 'csv'
 
 module DataShift
 
   class CsvExporter < ExporterBase
 
-    attr_accessor :excel, :filename
-  
+    attr_accessor :text_delim
+    
     def initialize(filename)
-      @excel = nil
-      @filename = filename
+      super(filename)
+      @text_delim = "\'"
     end
 
-    # Create CSV file representing supplied Model
     
-    def generate(model, options = {})
-
-      @filename = options[:filename] if  options[:filename]
+    # Return opposite of text delim - "hello, 'barry'" => '"hello, "barry""'
+    def escape_text_delim
+      return '"' if @text_delim == "\'"
+      "\'"
+    end
+    
+    # Create CSV file from set of ActiveRecord objects
+    # Options :
+    # => :filename
+    # => :text_delim => Char to use to delim columns, useful when data contain embedded ','
+    # => ::methods => List of methods to additionally call on each record
+    #
+    def export(records, options = {})
+       
+      raise ArgumentError.new('Please supply array of records to export') unless records.is_a? Array
+      
+      first = records[0]
+     
+      return unless(first.is_a?(ActiveRecord::Base))
+      
+      f = options[:filename] || filename()
+      
+      @text_delim = options[:text_delim] if(options[:text_delim])
+      
+      File.open(f, "w") do |csv|
+        
+        headers = first.serializable_hash.keys.collect
+           
+        [*options[:methods]].each do |c| headers << c if(first.respond_to?(c)) end if(options[:methods])
+             
+        csv << headers.join(Delimiters::csv_delim) << Delimiters::eol
+        
+        records.each do |r|
+          next unless(r.is_a?(ActiveRecord::Base))
+          
+          csv << record_to_csv(r, options) << Delimiters::eol
+        end
+      end
     end
+      
+    # Create an Excel file from list of ActiveRecord objects
+    # Specify which associations to export via :with or :exclude
+    # Possible values are : [:assignment, :belongs_to, :has_one, :has_many]
+    #
+    def export_with_associations(klass, records, options = {})
+        
+      f = options[:filename] || filename()
+      
+      @text_delim = options[:text_delim] if(options[:text_delim])
+       
+      MethodDictionary.find_operators( klass )
+        
+      # builds all possible operators
+      MethodDictionary.build_method_details( klass )
+           
+      work_list = options[:with] ? Set(options[:with]) : MethodDetail::supported_types_enum
+      
+      assoc_work_list = work_list.dup
 
+      details_mgr = MethodDictionary.method_details_mgrs[klass]
+
+      headers, assoc_operators, assignments = [], [], []
+      
+      # headers
+      if(work_list.include?(:assignment))
+        assignments << details_mgr.get_list(:assignment).collect( &:operator)
+        assoc_work_list.delete :assignment
+      end
+         
+      headers << assignments.flatten!
+      # based on users requested list ... belongs_to has_one, has_many etc ... select only those operators
+      assoc_operators = assoc_work_list.collect do |op_type|     
+        details_mgr.get_list(op_type).collect(&:operator).flatten
+      end
+      
+      assoc_operators.flatten!
+      
+      File.open(f, "w") do |csv|  
+          
+        csv << headers.join(Delimiters::csv_delim)
+        
+        csv << Delimiters::csv_delim << assoc_operators.join(Delimiters::csv_delim) unless(assoc_operators.empty?)
+        
+        csv << Delimiters::eol
+        
+        records.each do |r|
+          next unless(r.is_a?(ActiveRecord::Base))
+          
+          csv_columns = []
+          # need to make sure order matches headers
+          # could look at collection headers via serializable hash.keys and then use 
+          # csv << record_to_csv(r) ??
+          assignments.each {|x| csv << escape(r.send(x)) << Delimiters::csv_delim }
+                 
+          # done records basic attributes now deal with associations
+          
+          #assoc_work_list.each do |op_type| 
+           # details_mgr.get_operators(op_type).each do |operator| 
+          assoc_operators.each do |operator| 
+              assoc_object = r.send(operator) 
+              
+              if(assoc_object.is_a?ActiveRecord::Base)
+                column_text = record_to_column(assoc_object)     # belongs_to or has_one
+                
+                csv << "#{@text_delim}#{column_text}#{@text_delim}" << Delimiters::csv_delim
+                #csv << record_to_csv(r)
+                
+              elsif(assoc_object.is_a? Array)
+                items_to_s = assoc_object.collect {|x| record_to_column(x) }
+                
+                # create a single column
+                csv << "#{@text_delim}#{items_to_s.join(Delimiters::multi_assoc_delim)}#{@text_delim}" << Delimiters::csv_delim
+                
+              else
+                csv << Delimiters::csv_delim
+              end
+            #end
+          end
+          
+          csv << Delimiters::eol   # next record
+          
+        end
+      end
+    end
+    
+  
+    # Convert an AR instance to a single CSV column
   
-    # Create an Csv file representing supplied Model
+    def record_to_column(record) 
+    
+      csv_data = []
+      record.serializable_hash.each do |name, value|
+        value = 'nil' if value.nil?
+        text = value.to_s.gsub(@text_delim, escape_text_delim())
+        csv_data << "#{name.to_sym} => #{text}"
+      end
+      "#{csv_data.join(Delimiters::csv_delim)}"
+    end
+    
+    
+    # Convert an AR instance to a set of CSV columns
+    def record_to_csv(record, options = {})
+      csv_data = record.serializable_hash.values.collect { |value| escape(value) }
 
-    def export(items, options = {})
+      [*options[:methods]].each { |x| csv_data << escape(record.send(x)) if(record.respond_to?(x)) } if(options[:methods])
+      
+      csv_data.join( Delimiters::csv_delim )
     end
+    
+
+    private
 
+    def escape(value)
+      text = value.to_s.gsub(@text_delim, escape_text_delim())
+      
+      text = "#{@text_delim}#{text}#{@text_delim}" if(text.include?(Delimiters::csv_delim)) 
+      text
+    end
+    
   end
 end