datashift 0.11.1 → 0.12.0

data/Rakefile

@@ -40,7 +40,8 @@ Jeweler::Tasks.new do |gem|
   gem.email = "rubygems@autotelik.co.uk"
   gem.authors = ["Thomas Statter"]
   # dependencies defined in Gemfile
-  gem.files.exclude ['sandbox']
+  gem.files.exclude ['sandbox/**']
+  gem.files.exclude ['spec/**/*']
   
   gem.add_dependency 'spreadsheet'
   gem.add_dependency 'rubyzip'
@@ -86,7 +87,7 @@ task :build, :version do |t, args|
   puts "Installing version #{version}"
   
   gem = "#{DataShift.gem_name}-#{version}.gem"
-  cmd = "jruby -S gem install --no-ri --no-rdoc #{gem}"
+  cmd = "gem install --no-ri --no-rdoc #{gem}"
   system(cmd)
 end
   

data/VERSION

@@ -1 +1 @@
-0.11.1
+0.12.0

data/datashift.gemspec

@@ -1,15 +1,12 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
-# -*- encoding: utf-8 -*-
-
+require 'rake'
+  
 Gem::Specification.new do |s|
   s.name = "datashift"
-  s.version = "0.11.1"
+  s.version = "0.12.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Thomas Statter"]
-  s.date = "2012-10-23"
+  s.date = "2013-01-02"
   s.description = "Comprehensive tools to import/export between Excel/CSV and ActiveRecord databases, Rails apps, and any Ruby project."
   s.email = "rubygems@autotelik.co.uk"
   s.extra_rdoc_files = [
@@ -17,183 +14,23 @@ Gem::Specification.new do |s|
     "README.markdown",
     "README.rdoc"
   ]
-  s.files = [
-    ".document",
+  
+  s.test_files = FileList["{spec}/*"]
+  
+  s.files = FileList[
     "LICENSE.txt",
     "README.markdown",
     "README.rdoc",
     "Rakefile",
     "VERSION",
     "datashift.gemspec",
-    "lib/applications/apache_poi_extensions.rb",
-    "lib/applications/excel.rb",
-    "lib/applications/excel_base.rb",
-    "lib/applications/jexcel_file.rb",
-    "lib/applications/jexcel_file_extensions.rb",
-    "lib/applications/jruby/old_pre_proxy_jexcel_file.rb",
-    "lib/applications/jruby/word.rb",
-    "lib/applications/ruby_poi_translations.rb",
-    "lib/applications/spreadsheet_extensions.rb",
-    "lib/datashift.rb",
-    "lib/datashift/delimiters.rb",
-    "lib/datashift/exceptions.rb",
-    "lib/datashift/file_definitions.rb",
-    "lib/datashift/logging.rb",
-    "lib/datashift/mapping_file_definitions.rb",
-    "lib/datashift/method_detail.rb",
-    "lib/datashift/method_details_manager.rb",
-    "lib/datashift/method_dictionary.rb",
-    "lib/datashift/method_mapper.rb",
-    "lib/datashift/model_mapper.rb",
-    "lib/datashift/populator.rb",
-    "lib/datashift/querying.rb",
-    "lib/exporters/csv_exporter.rb",
-    "lib/exporters/excel_exporter.rb",
-    "lib/exporters/exporter_base.rb",
-    "lib/generators/csv_generator.rb",
-    "lib/generators/excel_generator.rb",
-    "lib/generators/generator_base.rb",
-    "lib/guards.rb",
-    "lib/helpers/core_ext/to_b.rb",
-    "lib/java/poi-3.7/._poi-3.7-20101029.jar5645100390082102460.tmp",
-    "lib/java/poi-3.7/LICENSE",
-    "lib/java/poi-3.7/NOTICE",
-    "lib/java/poi-3.7/RELEASE_NOTES.txt",
-    "lib/java/poi-3.7/lib/commons-logging-1.1.jar",
-    "lib/java/poi-3.7/lib/junit-3.8.1.jar",
-    "lib/java/poi-3.7/lib/log4j-1.2.13.jar",
-    "lib/java/poi-3.7/ooxml-lib/dom4j-1.6.1.jar",
-    "lib/java/poi-3.7/ooxml-lib/geronimo-stax-api_1.0_spec-1.0.jar",
-    "lib/java/poi-3.7/ooxml-lib/xmlbeans-2.3.0.jar",
-    "lib/java/poi-3.7/poi-3.7-20101029.jar",
-    "lib/java/poi-3.7/poi-examples-3.7-20101029.jar",
-    "lib/java/poi-3.7/poi-ooxml-3.7-20101029.jar",
-    "lib/java/poi-3.7/poi-ooxml-schemas-3.7-20101029.jar",
-    "lib/java/poi-3.7/poi-scratchpad-3.7-20101029.jar",
-    "lib/loaders/csv_loader.rb",
-    "lib/loaders/excel_loader.rb",
-    "lib/loaders/loader_base.rb",
-    "lib/loaders/paperclip/attachment_loader.rb",
-    "lib/loaders/paperclip/datashift_paperclip.rb",
-    "lib/loaders/paperclip/image_loading.rb",
-    "lib/thor/export.thor",
-    "lib/thor/generate.thor",
-    "lib/thor/import.thor",
-    "lib/thor/paperclip.thor",
-    "lib/thor/tools.thor",
-    "spec/Gemfile",
-    "spec/MissingAttachmentRecords/DEMO_001_ror_bag.jpeg",
-    "spec/MissingAttachmentRecords/DEMO_002_Powerstation.jpeg",
-    "spec/MissingAttachmentRecords/DEMO_002_Powerstation.jpg",
-    "spec/MissingAttachmentRecords/DEMO_003_ror_mug.jpeg",
-    "spec/MissingAttachmentRecords/DEMO_004_ror_ringer.jpeg",
-    "spec/csv_exporter_spec.rb",
-    "spec/csv_loader_spec.rb",
-    "spec/datashift_spec.rb",
-    "spec/excel_exporter_spec.rb",
-    "spec/excel_generator_spec.rb",
-    "spec/excel_loader_spec.rb",
-    "spec/excel_spec.rb",
-    "spec/file_definitions.rb",
-    "spec/fixtures/BadAssociationName.xls",
-    "spec/fixtures/DemoNegativeTesting.xls",
-    "spec/fixtures/ProjectsDefaults.yml",
-    "spec/fixtures/ProjectsMultiCategories.xls",
-    "spec/fixtures/ProjectsMultiCategoriesHeaderLookup.xls",
-    "spec/fixtures/ProjectsSingleCategories.xls",
-    "spec/fixtures/SimpleProjects.xls",
-    "spec/fixtures/config/database.yml",
-    "spec/fixtures/db/datashift_test_models_db.sqlite",
-    "spec/fixtures/db/migrate/20110803201325_create_test_bed.rb",
-    "spec/fixtures/db/migrate/20121009161700_add_digitals.rb",
-    "spec/fixtures/images/DEMO_001_ror_bag.jpeg",
-    "spec/fixtures/images/DEMO_002_Powerstation.jpeg",
-    "spec/fixtures/images/DEMO_003_ror_mug.jpeg",
-    "spec/fixtures/images/DEMO_004_ror_ringer.jpeg",
-    "spec/fixtures/load_datashift.thor",
-    "spec/fixtures/models/category.rb",
-    "spec/fixtures/models/digital.rb",
-    "spec/fixtures/models/empty.rb",
-    "spec/fixtures/models/loader_release.rb",
-    "spec/fixtures/models/long_and_complex_table_linked_to_version.rb",
-    "spec/fixtures/models/milestone.rb",
-    "spec/fixtures/models/owner.rb",
-    "spec/fixtures/models/project.rb",
-    "spec/fixtures/models/version.rb",
-    "spec/fixtures/simple_export_spec.xls",
-    "spec/fixtures/simple_template_spec.xls",
-    "spec/fixtures/test_model_defs.rb",
-    "spec/loader_spec.rb",
-    "spec/method_dictionary_spec.rb",
-    "spec/method_mapper_spec.rb",
-    "spec/paperclip_loader_spec.rb",
-    "spec/rails_sandbox/.gitignore",
-    "spec/rails_sandbox/Gemfile",
-    "spec/rails_sandbox/README.rdoc",
-    "spec/rails_sandbox/Rakefile",
-    "spec/rails_sandbox/app/assets/images/rails.png",
-    "spec/rails_sandbox/app/assets/javascripts/application.js",
-    "spec/rails_sandbox/app/assets/stylesheets/application.css",
-    "spec/rails_sandbox/app/controllers/application_controller.rb",
-    "spec/rails_sandbox/app/helpers/application_helper.rb",
-    "spec/rails_sandbox/app/mailers/.gitkeep",
-    "spec/rails_sandbox/app/models/.gitkeep",
-    "spec/rails_sandbox/app/models/category.rb",
-    "spec/rails_sandbox/app/models/empty.rb",
-    "spec/rails_sandbox/app/models/loader_release.rb",
-    "spec/rails_sandbox/app/models/long_and_complex_table_linked_to_version.rb",
-    "spec/rails_sandbox/app/models/milestone.rb",
-    "spec/rails_sandbox/app/models/owner.rb",
-    "spec/rails_sandbox/app/models/project.rb",
-    "spec/rails_sandbox/app/models/test_model_defs.rb",
-    "spec/rails_sandbox/app/models/version.rb",
-    "spec/rails_sandbox/app/views/layouts/application.html.erb",
-    "spec/rails_sandbox/config.ru",
-    "spec/rails_sandbox/config/application.rb",
-    "spec/rails_sandbox/config/boot.rb",
-    "spec/rails_sandbox/config/database.yml",
-    "spec/rails_sandbox/config/environment.rb",
-    "spec/rails_sandbox/config/environments/development.rb",
-    "spec/rails_sandbox/config/environments/production.rb",
-    "spec/rails_sandbox/config/environments/test.rb",
-    "spec/rails_sandbox/config/initializers/backtrace_silencers.rb",
-    "spec/rails_sandbox/config/initializers/inflections.rb",
-    "spec/rails_sandbox/config/initializers/mime_types.rb",
-    "spec/rails_sandbox/config/initializers/secret_token.rb",
-    "spec/rails_sandbox/config/initializers/session_store.rb",
-    "spec/rails_sandbox/config/initializers/wrap_parameters.rb",
-    "spec/rails_sandbox/config/locales/en.yml",
-    "spec/rails_sandbox/config/routes.rb",
-    "spec/rails_sandbox/db/migrate/20110803201325_create_test_bed.rb",
-    "spec/rails_sandbox/db/schema.rb",
-    "spec/rails_sandbox/db/seeds.rb",
-    "spec/rails_sandbox/lib/assets/.gitkeep",
-    "spec/rails_sandbox/lib/tasks/.gitkeep",
-    "spec/rails_sandbox/log/.gitkeep",
-    "spec/rails_sandbox/public/404.html",
-    "spec/rails_sandbox/public/422.html",
-    "spec/rails_sandbox/public/500.html",
-    "spec/rails_sandbox/public/favicon.ico",
-    "spec/rails_sandbox/public/index.html",
-    "spec/rails_sandbox/public/robots.txt",
-    "spec/rails_sandbox/script/rails",
-    "spec/rails_sandbox/test/fixtures/.gitkeep",
-    "spec/rails_sandbox/test/functional/.gitkeep",
-    "spec/rails_sandbox/test/integration/.gitkeep",
-    "spec/rails_sandbox/test/performance/browsing_test.rb",
-    "spec/rails_sandbox/test/test_helper.rb",
-    "spec/rails_sandbox/test/unit/.gitkeep",
-    "spec/rails_sandbox/vendor/assets/javascripts/.gitkeep",
-    "spec/rails_sandbox/vendor/assets/stylesheets/.gitkeep",
-    "spec/rails_sandbox/vendor/plugins/.gitkeep",
-    "spec/spec_helper.rb",
-    "spec/thor_spec.rb",
     "tasks/config/seed_fu_product_template.erb",
     "tasks/config/tidy_config.txt",
     "tasks/db_tasks.rake",
     "tasks/file_tasks.rake",
-    "tasks/word_to_seedfu.rake"
-  ]
+    "tasks/word_to_seedfu.rake",
+    "{lib}/**/*"].exclude("rdoc").exclude("nbproject").exclude("fixtures").exclude(".log").exclude(".contrib").to_a
+  
   s.homepage = "http://github.com/autotelik/datashift"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]

data/lib/datashift/column_packer.rb

@@ -0,0 +1,64 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Dec 2012
+# License::   MIT
+#
+# Details::   Helper for creating consistent import/export format
+#             of model's attributes/associations
+# 
+#
+require 'exporter_base'
+require 'csv'
+
+module DataShift
+
+  module ColumnPacker
+
+    
+    def text_delim
+      @text_delim ||= "\'"
+    end
+
+    def text_delim=(x)
+      @text_delim = x
+    end
+    
+    # Return opposite of text delim - "hello, 'barry'" => '"hello, "barry""'
+    def escape_text_delim
+      return '"' if text_delim == "\'"
+      "\'"
+    end
+    
+ 
+    # Convert an AR instance to a single column
+  
+    def record_to_column(record, attribute_delim = Delimiters::csv_delim) 
+    
+      csv_data = []
+      record.serializable_hash.each do |name, value|
+        value = 'nil' if value.nil?
+        text = value.to_s.gsub(@text_delim, escape_text_delim())
+        csv_data << "#{name.to_sym} => #{text}"
+      end
+      "#{csv_data.join(attribute_delim)}"
+    end
+    
+    
+    # Convert an AR instance to a set of CSV columns
+    def record_to_csv(record, options = {})
+      csv_data = record.serializable_hash.values.collect { |value| escape_for_csv(value) }
+
+      [*options[:methods]].each { |x| csv_data << escape_for_csv(record.send(x)) if(record.respond_to?(x)) } if(options[:methods])
+      
+      csv_data.join( Delimiters::csv_delim )
+    end
+    
+    def escape_for_csv(value)
+      text = value.to_s.gsub(@text_delim, escape_text_delim())
+      
+      text = "#{@text_delim}#{text}#{@text_delim}" if(text.include?(Delimiters::csv_delim)) 
+      text
+    end
+    
+  end
+end

data/lib/datashift/delimiters.rb

@@ -67,15 +67,31 @@ module DataShift
       @multi_assoc_delim
     end
 
-
     def self.set_multi_assoc_delim(x) @multi_assoc_delim = x; end
     
     
+    # Delimiters for {:abc => 2, :efg => 'some text}
+    
+    def self.attribute_list_start 
+      @attribute_list_start ||= '{';
+    end
+
+    def self.attribute_list_start=(x) @attribute_list_start = x; end
+    
+    def self.attribute_list_end
+      @attribute_list_end ||= '}'
+    end
+    
+    def self.attribute_list_end=(x) 
+      @attribute_list_end = x; 
+    end
+  
     def self.csv_delim
       @csv_delim ||= ','
       @csv_delim
     end
     
+    def self.csv_delim=(x) set_csv_delim(x); end
     def self.set_csv_delim(x) @csv_delim = x; end
     
     def self.eol

data/lib/datashift/exceptions.rb

@@ -6,10 +6,17 @@ module DataShift
   class BadFile < StandardError; end
   
   class MappingDefinitionError < StandardError; end
+  class DataProcessingError < StandardError; end
 
   class MissingHeadersError < StandardError; end
   class MissingMandatoryError < StandardError; end
 
   class RecordNotFound < StandardError; end
   
+  class PathError < StandardError; end
+  
+  class BadUri < StandardError; end
+   
+  class CreateAttachmentFailed < StandardError; end
+  
 end

data/lib/datashift/querying.rb

@@ -18,11 +18,9 @@ module DataShift
  
     # Options:
     # 
-    #   :split_on_prefix  : Add a prefix to each search term
     #   :case_sensitive   : Default is a case insensitive lookup.
     #   :use_like         : Attempts a lookup using ike and x% rather than equality 
     #
-    
     def search_for_record(klazz, field, search_term, options = {})
     
       begin
@@ -50,7 +48,7 @@ module DataShift
     # 
     #   :add_prefix     : Add a prefix to each search term
     #   :case_sensitive : Default is a case insensitive lookup.
-    #   :use_like       : Attempts a lookup using ike and x% rather than equality 
+    #   :use_like       : Attempts a lookup using like and x% rather than equality 
     #
     # Returns nil if no record found
     def get_record_by(klazz, field, search_term, split_on = ' ', options = {})
@@ -59,7 +57,11 @@ module DataShift
          
         split_on_prefix = options[:add_prefix]
         
-        record = search_for_record(klazz, field, search_term)
+        z = (split_on_prefix) ? "#{split_on_prefix}#{search_term}": search_term
+        
+        logger.info("Scanning for record where #{klazz}.#{field} ~=  #{z}")
+        
+        record = search_for_record(klazz, field, z)
         
         # try individual portions of search_term, front -> back i.e "A_B_C_D" => A, B, C etc
         search_term.split(split_on).each do |str|
@@ -76,6 +78,10 @@ module DataShift
           term
         end unless(record)
         
+        if(record && record.respond_to?(field)) 
+          logger.info("Record found for #{klazz}.#{field} : #{record.send(field)}" )  
+        end
+        
         return record
       rescue => e
         logger.error("Exception attempting to find a record for [#{search_term}] on #{klazz}.#{field}")

data/lib/loaders/csv_loader.rb

@@ -15,9 +15,16 @@ module DataShift
   module CsvLoading
     
     include DataShift::Logging
-    
-    #   Assumes header_row is first row i.e row 0
+
+    # Load data through active Record models into DB from a CSV file
+    #  
+    # Assumes header_row is first row i.e row 0
     #   
+    #   
+    # OPTIONS :
+    #  
+    #  [:dummy]           : Perform a dummy run - attempt to load everything but then roll back
+    #
     #  Options passed through  to :  populate_method_mapper_from_headers
     #  
     #   [:mandatory]       : Array of mandatory column names
@@ -40,45 +47,75 @@ module DataShift
 
       puts "\n\n\nLoading from CSV file: #{file_name}"
       puts "Processing #{@parsed_file.size} rows"
+      begin
+  
+        load_object_class.transaction do
+          @reporter.reset
 
-      load_object_class.transaction do
-        @loaded_objects =  []
-
-        @parsed_file.each do |row|
-          
-          # First assign any default values for columns not included in parsed_file
-          process_missing_columns_with_defaults
+          @parsed_file.each_with_index do |row, i|
+            
+            @current_row = row 
+            
+            @reporter.processed_object_count += 1
+            
+            begin
+              # First assign any default values for columns not included in parsed_file
+              process_missing_columns_with_defaults
 
-          # TODO - Smart sorting of column processing order ....
-          # Does not currently ensure mandatory columns (for valid?) processed first but model needs saving
-          # before associations can be processed so user should ensure mandatory columns are prior to associations
+              # TODO - Smart sorting of column processing order ....
+              # Does not currently ensure mandatory columns (for valid?) processed first but model needs saving
+              # before associations can be processed so user should ensure mandatory columns are prior to associations
 
-          # as part of this we also attempt to save early, for example before assigning to
-          # has_and_belongs_to associations which require the load_object has an id for the join table
+              # as part of this we also attempt to save early, for example before assigning to
+              # has_and_belongs_to associations which require the load_object has an id for the join table
 
-          # Iterate over the columns method_mapper found in Excel,
-          # pulling data out of associated column
-          @method_mapper.method_details.each_with_index do |method_detail, col|
+              # Iterate over the columns method_mapper found in Excel,
+              # pulling data out of associated column
+              @method_mapper.method_details.each_with_index do |method_detail, col|
 
-            value = row[col]
+                value = row[col]
 
-            prepare_data(method_detail, value)
+                prepare_data(method_detail, value)
             
-            process()
-          end
-
-          # TODO - handle when it's not valid ?
-          # Process rest and dump out an exception list of Products ??
-
-          logger.info "Saving csv row #{row} to table object : #{load_object.inspect}"
-
-          save
-
-          # don't forget to reset the object or we'll update rather than create
-          new_load_object
+                process()
+              end
+
+            rescue => e
+              failure( row, true )
+              logger.error "Failed to process row [#{i}] (#{@current_row})"
+              # don't forget to reset the load object 
+              new_load_object
+              next
+            end
+            
+            # TODO - make optional -  all or nothing or carry on and dump out the exception list at end        
+            unless(save)
+              failure
+              logger.error "Failed to save row [#{@current_row}] (#{load_object.inspect})"
+              logger.error load_object.errors.inspect if(load_object)
+            else
+              logger.info "Row #{@current_row} succesfully SAVED : ID #{load_object.id}"
+              @reporter.add_loaded_object(@load_object)
+            end
+
+            # don't forget to reset the object or we'll update rather than create
+            new_load_object
 
+          end
+        
+          raise ActiveRecord::Rollback if(options[:dummy]) # Don't actually create/upload to DB if we are doing dummy run
         end
+      rescue => e
+        puts "CAUGHT ", e.backtrace, e.inspect
+        if e.is_a?(ActiveRecord::Rollback) && options[:dummy]
+          puts "CSV loading stage complete - Dummy run so Rolling Back."
+        else
+          raise e
+        end
+      ensure
+        report
       end
+    
     end
   end
   
@@ -94,7 +131,7 @@ module DataShift
     def perform_load( file_name, options = {} )
       perform_csv_load( file_name, options )
 
-      puts "CSV loading stage complete - #{loaded_objects.size} rows added."
+      puts "CSV loading stage complete - #{loaded_count} rows added."
     end
 
   end