datashift 0.16.0 → 0.40.0

data/lib/loaders/loader_factory.rb

@@ -0,0 +1,42 @@
+# Copyright:: (c) Autotelik Media Ltd 2011
+# Author ::   Tom Statter
+# Date ::     Aug 2010
+# License::   MIT
+#
+#  Details::  Module for loaders, providing a process hook which populates a model,
+#             based on a method map and supplied value from a file - i.e a single column/row's string value.
+#             Note that although a single column, the string can be formatted to contain multiple values.
+#
+#             Tightly coupled with Binder classes (in lib/engine) which contains full details of
+#             a file's column and it's correlated AR associations.
+#
+module DataShift
+
+  module Loader
+
+    class Factory
+
+      # Based on file_name find appropriate Loader
+
+      # Currently supports :
+      #   Excel/Open Office files saved as .xls
+      #   CSV files
+      #
+      def self.get_loader(file_name)
+
+        raise DataShift::BadFile, "Cannot load #{file_name} file not found." unless File.exist?(file_name)
+
+        ext = File.extname(file_name)
+
+        if ext.casecmp('.xls') == 0 || ext.casecmp('.xlsx') == 0
+          return ExcelLoader.new
+        elsif ext.casecmp('.csv') == 0
+          return CsvLoader.new
+        else
+          raise DataShift::UnsupportedFileType, "#{ext} files not supported - Try .csv or OpenOffice/Excel .xls"
+        end
+      end
+
+    end
+  end
+end

data/lib/loaders/paperclip/attachment_loader.rb

@@ -9,176 +9,193 @@ require 'loader_base'
 require 'datashift_paperclip'
 
 module DataShift
- 
+
   module Paperclip
-    
+
     class AttachmentLoader < LoaderBase
-      
+
       include DataShift::Paperclip
-      
-      attr_accessor :attach_to_klass, :attach_to_find_by_field
-      
-      attr_writer :attach_to_field
-      attr_reader :attachment_path, :loading_files_cache
-
-      
-      # Constructor
-      # 
-      # Options 
-      #
-      # => :attach_to_klass    
-      #       A class that has a relationship with the attachment (has_many, has_one or belongs_to etc)  
+
+      attr_accessor :attach_to_klass, :attach_to_find_by_field, :attach_to_field
+
+      # We try splitting up inbound file_names in various ways looking for the attachment Owner
+      attr_accessor :split_file_name_on
+
+      attr_reader :loading_files_cache, :missing_records
+
+      def initialize
+        super
+
+        @attach_to_klass = nil
+        @attach_to_find_by_field = nil
+        @attach_to_field = nil
+
+        @split_file_name_on = Regexp.new(/\s+/)
+
+        @missing_records = []
+      end
+
+      # => :attach_to_klass
+      #       A class that has a relationship with the attachment (has_many, has_one or belongs_to etc)
       #       The instance of :attach_to_klass can be searched for and the new attachment assigned.
-      #       
-      #     Examples     
+      #
+      #     Examples
       #       Owner has_many pdfs and mp3 files as Digitals .... :attach_to_klass = Owner
       #       User has a single image used as an avatar ... :attach_to_klass = User
       #
-      # => :attach_to_find_by_field    
-      #       For the :attach_to_klass, this is the field used to search for the parent
-      #       object to assign the new attachment to.
-      #       
-      #     Examples     
+      # => :attach_to_find_by_field
+      #       Field on the :attach_to_klass, this is the field used to search for the
+      #       object (class == attach_to_klass) to assign the new attachment to.
+      #
+      #     Examples
       #       Owner has a unique 'name' field ... :attach_to_find_by_field = :name
       #       User has a unique  'login' field  ... :attach_to_klass = :login
       #
-      # => :attach_to_field    
+      # => :attach_to_field
       #       Attribute/association to assign attachment to on :attach_to_klass.
       #      Examples
-      #      
+      #
       #         :attach_to_field => digitals  : Owner.digitals = attachment
       #         :attach_to_field => avatar    : User.avatar = attachment
-      #         
-      #       
-      def initialize(attachment_klazz, attachment = nil, options = {})
-        
-        init_from_options( options )
- 
-        super( attachment_klazz, attachment, options.dup )
-         
-        puts "Attachment Class is #{load_object_class}" if(@verbose)
-       
-        raise "Failed to create Attachment for loading" unless @load_object
-      end
-            
-      
-      # Options
-      # :reload
-      # :attach_to_klass, :attach_to_field, :attach_to_find_by_field
       #
-      def init_from_options( options )
-        
-        @attach_to_klass  = options[:attach_to_klass] || @attach_to_klass || nil
-            
-        unless(@attach_to_klass.nil? || (MethodDictionary::for?(@attach_to_klass) && options[:reload] == false))
-          #puts "Building Method Dictionary for class #{object_class}"
-          DataShift::MethodDictionary.find_operators( @attach_to_klass, :reload => options[:reload], :instance_methods => true )
-        
-          # Create dictionary of data on all possible 'setter' methods which can be used to
-          # populate or integrate an object of type @load_object_class
-          DataShift::MethodDictionary.build_method_details(@attach_to_klass)
-        end
-      
-        @attach_to_find_by_field  = options[:attach_to_find_by_field] || @attach_to_find_by_field || nil
-        @attach_to_field  = options[:attach_to_field] || @attach_to_field || nil
-        
-        unless(@attach_to_klass.nil? && @attach_to_field.nil? )
-          @attach_to_method_detail = MethodDictionary.find_method_detail(@attach_to_klass, @attach_to_field)
+      def init(attach_to_klass, attach_to_find_by_field, attach_to_field, options = {})
+
+        @attach_to_klass = MapperUtils.ensure_class( attach_to_klass )
+
+        ModelMethods::Manager.catalog_class(@attach_to_klass, reload: options[:reload], instance_methods: true)
+
+        @attach_to_find_by_field = attach_to_find_by_field
+        @attach_to_field = attach_to_field
+      end
+
+      def init_from_options(options)
+        init(options[:attach_to_klass], options[:attach_to_find_by_field], options[:attach_to_field])
+
+        @split_file_name_on = options[:split_file_name_on] if(options[:split_file_name_on])
+      end
+
+      def find_bindings
+        # Map field to a suitable call on the Active Record Owner class e.g Owner.digitals
+        bindings = begin
+          logger.info("Finding matching field/association [#{attach_to_field}] on class [#{attach_to_klass}]")
+
+          binder.map_inbound_fields(attach_to_klass, attach_to_field)
+        rescue => e
+          logger.error("Failed to map #{attach_to_field} to database operator : #{e.inspect}")
+          logger.error( e.backtrace )
+          raise MappingDefinitionError, 'Failed to map #{attach_to_field} to database operator'
         end
+
+        bindings
       end
-      
-            
-      # If we have instantiated a method detail based on the attach to class and fields
-      # return that otherwise return the raw format of :attach_to_find_by_field
-      
-      def attach_to_field
-        @attach_to_method_detail || @attach_to_field
+
+      def find_owner(in_file_name, options = {} )
+
+        search_term = File.basename(in_file_name, '.*')
+        search_term.strip!
+
+        logger.info("Attempting to find attachment owner (#{attach_to_klass} for [#{search_term}]")
+
+        owner_record = get_record_by(attach_to_klass, attach_to_find_by_field, search_term, split_file_name_on, options)
+
+        if owner_record
+          logger.info("#{owner_record.class} (id : #{owner_record.id}) found with matching :#{attach_to_find_by_field} ")
+        else
+
+          message = "No matching owner found for file name : #{search_term}"
+
+          failed = FailureData.new(nil, doc_context.node_context, message)
+
+          doc_context.progress_monitor.failure(failed)
+
+          missing_records << in_file_name
+        end
+
+        owner_record
       end
-      
-      
+
       # This version creates attachments and also attaches them to instances of :attach_to_klazz
-      # 
-      # Each file found in PATH will be processed - it's filename being used to scan for
+      #
+      # Each file found in PATH will be processed - it's file_name being used to scan for
       # a matching record to attach the file to.
-      # 
+      #
       # Options
-      #   :split_file_name_on   Used in scan process to progresivly split filename to find   
-      #                         :attach_to_klass with matching :attach_to_find_by_field
+      #   :split_file_name_on   Used in scan process to progressively split file_name to find
       #
       #   :add_prefix
       #
-      def process_from_filesystem(path, options = {} )
-     
-        init_from_options( options )
-        
-        raise "The class that attachments belongs to has not been set (:attach_to_klass)" unless(@attach_to_klass) 
-                
-        raise "The field to search for attachment's owner has not been set (:attach_to_find_by_field)" unless(@attach_to_find_by_field)
-               
-        @load_object = options[:attachment] if(options[:attachment])
-  
-        @attachment_path = path
-        
-        missing_records = []
-         
-        # try splitting up filename in various ways looking for the attachment owqner
-        split_on  = @config['split_file_name_on'] || options[:split_file_name_on]
-             
-        @loading_files_cache = DataShift::Paperclip::get_files(path, options)
-      
-        puts "Found #{loading_files_cache.size} files - splitting names on delimiter [#{split_on}]"
-
-        loading_files_cache.each do |file_name|
-
-          attachment_name = File.basename(file_name)
+      def perform_load(options = {} )
+
+        raise 'The class that attachments belongs to has not been set (:attach_to_klass)' unless @attach_to_klass
+
+        raise "The field to search for attachment's owner has not been set (:attach_to_find_by_field)" unless @attach_to_find_by_field
+
+        @missing_records = []
+
+        @load_object = options[:attachment] if options[:attachment]
+
+        # Support both directory and file
+        @loading_files_cache = DataShift::Paperclip.get_files(file_name, options)
+
+        logger.info("Found #{loading_files_cache.size} files - splitting names on delimiter [#{split_file_name_on}]")
+
+        # Map field to a suitable call on the Active Record Owner class e.g Owner.digitals
+        bindings = find_bindings
+
+        attach_to_method_binding = if bindings.size != 1
+                                     logger.warn("Failed to map #{attach_to_field} to database operator")
+                                     nil
+                                   else
+                                     bindings[0]
+                                   end
+
+        # Iterate through all the files creating an attachment per file
+
+        loading_files_cache.each do |in_file_name|
+          attachment_name = File.basename(in_file_name)
 
           logger.info "Processing attachment file #{attachment_name} "
-          
-          base_name = File.basename(file_name, '.*')
-          base_name.strip!
-            
-          record = nil
-            
-          puts "Attempting fo find Record for file name : #{base_name}"
-          record = get_record_by(attach_to_klass, attach_to_find_by_field, base_name, split_on, options)
-             
-          if(record)
-            puts "Found #{record.class} where : #{attach_to_find_by_field} = #{record.send(attach_to_find_by_field)}(id : #{record.id})"
-          else
-            missing_records << file_name
-          end
-          
-          next if(options[:dummy]) # Don't actually create/upload to DB if we are doing dummy run
-
-          # Check if attachment must have an associated record
-          if(record)
-            reset()
-
-            create_paperclip_attachment(@load_object_class, file_name, record, attach_to_field, options)
-   
-            puts "Added Attachment #{File.basename(file_name)} to #{record.send(attach_to_find_by_field)}(id : #{record.id})" if(@verbose)
-          end
 
-        end
+          owner_record = find_owner(in_file_name, options)
+
+          next if(configuration.dummy_run) # Don't actually create/upload to DB if we are doing dummy run
+
+          attachment = create_paperclip_attachment(load_object_class, in_file_name, options)
+
+          # Check if attachment must have an associated owner_record
+          next unless attachment && owner_record && attach_to_method_binding
+          reset
 
-        unless missing_records.empty?
-          FileUtils.mkdir_p('MissingAttachmentRecords') unless File.directory?('MissingAttachmentRecords')
-        
-          puts "WARNING : #{missing_records.size} of #{loading_files_cache.size} files could not be attached to a #{@load_object_class}"
-          puts "For your convenience copying files with MISSING #{attach_to_klass} to : MissingAttachmentRecords"
-          missing_records.each do |i| 
-            FileUtils.cp( i, 'MissingAttachmentRecords')  unless(options[:dummy] == 'true')
-            puts "Copyied #{i} to MissingAttachmentRecords folder" if(options[:verbose])
-          end
+          # TOFIX - what about has_one etc ? - indicates that Context etc are still too complex and Excel/CSV focused
+          owner_record.send(attach_to_method_binding.operator) << attachment
+
+          logger.info "Added Attachment to #{owner_record.class} (id : #{owner_record.id})"
         end
 
-        puts "Created #{loading_files_cache.size - missing_records.size} of #{loading_files_cache.size} #{@load_object_class} attachments and succesfully attached to a #{@attach_to_klass}"
-         
-        puts "Dummy Run Complete- if happy run without -d" if(options[:dummy])
-   
+        process_missing_records( missing_records )
+
+        created = loading_files_cache.size - missing_records.size
+
+        puts "Created #{created} / #{loading_files_cache.size} attachments of type #{load_object_class} attached to #{@attach_to_klass}"
+
+        puts 'Dummy Run Complete - if happy run with full commit' if(configuration.dummy_run)
+
+      end
+
+    end
+
+    def process_missing_records( missing_records )
+      unless missing_records.empty?
+        FileUtils.mkdir_p('MissingAttachmentRecords') unless File.directory?('MissingAttachmentRecords')
+
+        puts "WARNING : #{missing_records.size} of #{loading_files_cache.size} files could not be attached to a #{load_object_class}"
+        puts "For your convenience copying files with MISSING #{attach_to_klass} to : MissingAttachmentRecords"
+        missing_records.each do |i|
+          logger.info("Copying #{i} to MissingAttachmentRecords folder")
+          FileUtils.cp( i, 'MissingAttachmentRecords') unless(configuration.dummy_run)
+        end
       end
-   
     end
-      
+
   end
-end
+end

data/lib/loaders/paperclip/datashift_paperclip.rb

@@ -24,24 +24,24 @@ module DataShift
     # =>  :recursive : Descend tree looking for files rather than just supplied path
 
     def self.get_files(path, options = {})
-      return [path] if(File.file?(path))
+      return [path] if File.file?(path)
       glob = options[:glob] ? options[:glob] : '*.*'
-      glob = (options['recursive'] || options[:recursive])  ? "**/#{glob}" : glob
+      glob = (options['recursive'] || options[:recursive]) ? "**/#{glob}" : glob
 
       Dir.glob("#{path}/#{glob}", File::FNM_CASEFOLD)
     end
 
     def get_file( attachment_path )
 
-      unless File.exists?(attachment_path) && File.readable?(attachment_path)
+      unless File.exist?(attachment_path) && File.readable?(attachment_path)
         logger.error("Cannot process Image from #{Dir.pwd}: Invalid Path #{attachment_path}")
         raise PathError.new("Cannot process Image : Invalid Path #{attachment_path}")
       end
 
       file = begin
-        File.new(attachment_path, "rb")
+        File.new(attachment_path, 'rb')
       rescue => e
-        puts e.inspect
+        logger.error(e.inspect)
         raise PathError.new("ERROR : Failed to read image from #{attachment_path}")
       end
 
@@ -72,55 +72,38 @@ module DataShift
     #
     #       Give : { :has_attached_file_attribute => :icon }
     #
-    def create_paperclip_attachment(klass, attachment_path, record = nil, attach_to_record_field = nil, options = {})
+    def create_paperclip_attachment(klass, attachment_path, options = {})
 
       logger.info("Paperclip::create_paperclip_attachment on Class #{klass}")
 
       has_attached_file_attribute = options[:has_attached_file_name] ? options[:has_attached_file_name].to_sym : :attachment
 
-      # e.g  (:attachment => File.read)
-
       attachment_file = get_file(attachment_path)
-      paperclip_attributes = { has_attached_file_attribute => attachment_file }
 
-      paperclip_attributes.merge!(options[:attributes]) if(options[:attributes])
+      paperclip_attributes = { "#{has_attached_file_attribute}": attachment_file}
+
+      paperclip_attributes.merge!(options[:attributes]) if options[:attributes]
 
       begin
-        @attachment = klass.new(paperclip_attributes, :without_protection => true)
+        @attachment = klass.new(paperclip_attributes)
       rescue => e
         logger.error( e.backtrace)
-        logger.error("Failed to create PaperClip Attachment for cl;ass #{klass} : #{e.inspect}")
-        raise CreateAttachmentFailed.new("Failed to create PaperClip Attachment from : #{attachment_path}")
+        raise CreateAttachmentFailed.new("Failed [#{e.message}] creating PaperClip Attachment on #{klass} for [#{attachment_path}]")
       ensure
         attachment_file.close unless attachment_file.closed?
       end
 
-      begin
-
-        if(@attachment.save)
-          puts "Success: Created Attachment #{@attachment.id} : #{@attachment.attachment_file_name}"
-
-          if(attach_to_record_field.is_a? MethodDetail)
-            DataShift::Populator.new().prepare_and_assign(attach_to_record_field, record, @attachment)
-          else
-            # assume its not a has_many and try basic send
-            record.send("#{attach_to_record_field}=", @attachment)
-          end if(record && attach_to_record_field)
-
-        else
-          puts "ERROR : Problem saving to DB : #{@attachment.inspect}"
-          puts @attachment.errors.messages.inspect
-        end
+      if @attachment.save
+        logger.info("Success: Created Attachment #{@attachment.id} : #{@attachment.attachment_file_name}")
 
         @attachment
-      rescue => e
-        logger.error("Problem saving Paperclip Attachment: #{e.inspect}")
-        puts e.inspect
-        raise CreateAttachmentFailed.new("PaperClip error - Problem saving Attachment")
-      ensure
-        attachment_file.close unless attachment_file.closed?
+      else
+        logger.error('Problem creating and saving Paperclip Attachment')
+        logger.error(@attachment.errors.messages.inspect)
+        raise CreateAttachmentFailed.new('PaperClip error - Problem saving Attachment')
       end
     end
+
   end
 
 end