datashift 0.9.0 → 0.10.0

data/lib/loaders/paperclip/attachment_loader.rb

@@ -0,0 +1,104 @@
+# Copyright:: (c) Autotelik Media Ltd 2012
+# Author ::   Tom Statter
+# Date ::     Sept 2012
+# License::   MIT. Free, Open Source.
+#
+# => Provides facilities for bulk uploading/exporting attachments provided by PaperClip gem
+#
+require 'loader_base'
+require 'datashift_paperclip'
+
+module DataShift
+ 
+  class AttachmentLoader < LoaderBase
+      
+    include DataShift::Paperclip
+      
+    attr_accessor :attach_to_klass
+      
+    def initialize(attachment_klazz, attachment = nil, options = {})
+        
+      opts = options.merge(:load => false) 
+
+      super( attachment_klazz, attachment, opts )
+        
+      @attach_to_klass  = options[:attach_to_klass]
+           
+      puts "Attachment Class is #{@attachment_klazz}" if(@verbose)
+          
+      raise "Failed to create Attachment for loading" unless @load_object
+    end
+      
+    # :split_file_name_on
+      
+    def process_from_filesystem(path, options )
+       
+      @attach_to_klass  = options[:attach_to_klazz] if(options[:attach_to_klazz])
+       
+      raise "The class that attachments belong to has not set" unless(@attach_to_klass)
+        
+      @attachment_path = path
+        
+      missing_records = []
+         
+      # try splitting up filename in various ways looking for the SKU
+      split_search_term  = @config['split_file_name_on'] || options[:split_file_name_on]
+             
+      cache = Paperclip::get_files(@attachment_path, options)
+      
+      puts "Found #{cache.size} files - splitting names on delimiter [#{split_search_term}]"
+       
+      lookup_field = options[:attach_to_lookup_field]
+        
+      cache.each do |file_name|
+
+        attachment_name = File.basename(file_name)
+
+        logger.info "Processing attachment file #{attachment_name} "
+          
+        base_name = File.basename(file_name, '.*')
+        base_name.strip!
+            
+        record = nil
+                   
+        record = get_record_by(@attach_to_klass, lookup_field, base_name, split_search_term)
+             
+        if(record)
+          logger.info "Found record for attachment : #{record.inspect}"
+        else
+          missing_records << file_name
+        end
+          
+        next if(options[:dummy]) # Don't actually create/upload to DB if we are doing dummy run
+
+        # Check if attachment must have an associated record
+        if(record)
+          reset()
+          
+          create_attachment(@load_object_class, file_name, record, options[:attach_to_klass_field], options)
+   
+          puts "Added Attachment #{File.basename(file_name)} to #{record.send(lookup_field)} : #{record.id}" if(@verbose)
+        end
+
+      end
+
+      unless missing_records.empty?
+        FileUtils.mkdir_p('MissingAttachmentRecords') unless File.directory?('MissingAttachmentRecords')
+        
+        puts "WARNING : #{missing_records.size} of #{cache.size} files could not be attached to a #{@load_object_class}"
+        puts "For your convenience a copy of files with MISSING #{@load_object_class} :  ./MissingAttachmentRecords"
+        missing_records.each do |i|
+          puts "Copying #{i} to MissingAttachmentRecords folder" if(options[:verbose])
+          FileUtils.cp( i, 'MissingAttachmentRecords')  unless(options[:dummy] == 'true')
+        end
+      else
+        puts "All files (#{cache.size}) were succesfully attached to a #{@load_object_class}"
+      end
+
+      puts "Dummy Run Complete- if happy run without -d" if(options[:dummy])
+   
+    end
+   
+  end
+      
+end

data/lib/loaders/paperclip/datashift_paperclip.rb

@@ -0,0 +1,78 @@
+# Copyright:: (c) Autotelik Media Ltd 2012
+# Author ::   Tom Statter
+# Date ::     Sept 2012
+# License::   MIT. Free, Open Source.
+#
+# Details::   Module containing common functionality for working with Paperclip attachments
+# 
+require 'logging'
+
+module DataShift
+
+  module Paperclip
+    
+    include DataShift::Logging
+    
+    # Get all image files (based on file extensions) from supplied path.
+    # Options : 
+    #     :glob : The glob to use to find files
+    # =>  :recursive : Descend tree looking for files rather than just supplied path
+    
+    def self.get_files(path, options = {})
+      glob = options[:glob] ? options[:glob] : '*.*'
+      glob = (options['recursive'] || options[:recursive])  ? "**/#{glob}" : glob
+      
+      Dir.glob("#{path}/#{glob}", File::FNM_CASEFOLD)
+    end
+  
+    def get_file( attachment_path )
+      
+      unless File.exists?(attachment_path) && File.readable?(attachment_path)
+        logger.error("Cannot process Image from #{Dir.pwd}: Invalid Path #{attachment_path}")
+        raise "Cannot process Image : Invalid Path #{attachment_path}"
+      end
+     
+      file = begin
+        File.new(attachment_path, "rb")
+      rescue => e
+        puts e.inspect
+        raise "ERROR : Failed to read image #{attachment_path}"
+      end
+      
+      file
+    end
+    
+    # Note the paperclip attachment model defines the storage path via something like :
+    # => :path => ":rails_root/public/blah/blahs/:id/:style/:basename.:extension"
+    # Options 
+    #   has_attached_file_name : Paperclip attachment name defined with macro 'has_attached_file :name'  
+    #     e.g 
+    #     has_attached_file :avatar =>  options[:has_attached_file_name] = :avatar
+    #     has_attached_file :icon   =>  options[:has_attached_file_name] = :icon
+    #
+    #   alt : Alternatice text for images
+    
+    def create_attachment(klass, attachment_path, record = nil, attach_to_record_field = nil, options = {})
+       
+      has_attached_file = options[:has_attached_file_name] ? options[:has_attached_file_name].to_sym : :attachment
+      
+      file = get_file(attachment_path)
+
+      begin
+        
+        attachment = if(record && attach_to_record_field)
+          klass.new( {has_attached_file => file}, :without_protection => true)  
+        else
+          klass.new( {has_attached_file => file, attach_to_record_field.to_sym => record}, :without_protection => true)  
+        end
+        puts attachment.save ? "Success: Created #{attachment.id} : #{attachment.attachment_file_name}" : "ERROR : Problem saving to DB : #{attachment.inspect}"
+      rescue => e
+        puts "PaperClip error - Problem creating Attachment from : #{attachment_path}"
+        puts e.inspect, e.backtrace
+      end
+    end
+    
+  end
+  
+  
+end

data/lib/loaders/paperclip/{image_loader.rb → image_loading.rb}

@@ -5,13 +5,13 @@
 #
 # => Provides facilities for bulk uploading/exporting attachments provided by PaperClip 
 # gem
-require 'loader_base'
+require 'datashift_paperclip'
 
 module DataShift
 
   module ImageLoading
  
-    include DataShift::Logging
+    include DataShift::Paperclip
     
     # Get all image files (based on file extensions) from supplied path.
     # Options : 
@@ -25,22 +25,6 @@ module DataShift
       Dir.glob("#{path}/#{glob}", File::FNM_CASEFOLD)
     end
   
-    def get_file( attachment_path )
-      
-      unless File.exists?(attachment_path) && File.readable?(attachment_path)
-        logger.error("Cannot process Image from #{Dir.pwd}: Invalid Path #{attachment_path}")
-        raise "Cannot process Image : Invalid Path #{attachment_path}"
-      end
-     
-      file = begin
-        File.new(attachment_path, "rb")
-      rescue => e
-        puts e.inspect
-        raise "ERROR : Failed to read image #{attachment_path}"
-      end
-      
-      file
-    end
     
     # Note the paperclip attachment model defines the storage path via something like :
     # => :path => ":rails_root/public/blah/blahs/:id/:style/:basename.:extension"

data/lib/thor/{generate_excel.thor → generate.thor}

@@ -72,6 +72,54 @@ module Datashift
       end
 
     end
+    
+    
+    desc "csv", "generate a template from an active record model (with optional associations)" 
+    method_option :model, :aliases => '-m', :required => true, :desc => "The active record model to export"
+    method_option :result, :aliases => '-r', :required => true, :desc => "Create template of model in supplied file"
+    method_option :assoc, :aliases => '-a', :type => :boolean, :desc => "Include all associations in the template"
+    method_option :exclude, :aliases => '-e',  :type => :array, :desc => "Use with -a : Exclude association types. Any from #{DataShift::MethodDetail::supported_types_enum.to_a.inspect}"
+    
+    def csv()
+     
+      # TODO - We're assuming run from a rails app/top level dir...
+      # ...can we make this more robust ? e.g what about when using active record but not in Rails app, 
+      require File.expand_path('config/environment.rb')
+   
+      require 'csv_generator'
+
+      model = options[:model]
+      result = options[:result]
+     
+      logger.info "Datashift: Start CSV template generation in #{result}"
+            
+      begin
+        # support modules e.g "Spree::Property") 
+        klass = ModelMapper::class_from_string(model)  #Kernel.const_get(model)
+      rescue NameError => e
+        puts e
+        raise Thor::Error.new("ERROR: No such Model [#{model}] found - check valid model supplied")
+      end
+
+      raise Thor::Error.new("ERROR: No such Model [#{model}] found - check valid model supplied") unless(klass)
+      
+      begin
+        gen = DataShift::CsvGenerator.new(result)
+
+        if(options[:assoc])
+          opts = (options[:exclude]) ? {:exclude => options[:exclude]} : {}
+          logger.info("Datashift: Generating with associations")
+          gen.generate_with_associations(klass, opts)
+        else
+          gen.generate(klass)
+        end
+      rescue => e
+        puts e
+        puts e.backtrace
+        puts "Warning: Error during generation, template may be incomplete"
+      end
+
+    end
   end
 
 end

data/lib/thor/paperclip.thor

@@ -0,0 +1,85 @@
+# Copyright:: (c) Autotelik Media Ltd 2012
+# Author ::   Tom Statter
+# Date ::     Sept 2012
+# License::   MIT.
+#
+# Usage::
+#
+#  To pull Datashift commands into your main application :
+#
+#     require 'datashift'
+#
+#     DataShift::load_commands
+#
+#     thor help datashift:paperclip:attach
+#
+require 'datashift'
+
+# Note, not DataShift, case sensitive, create namespace for command line : datashift
+module Datashift
+         
+  class Paperclip < Thor     
+  
+    include DataShift::Logging   
+
+    desc "attach", "Attach files from a directory\nwhere file names contain somewhere the lookup info"
+    
+    # :dummy => dummy run without actual saving to DB
+    method_option :input, :aliases => '-i', :required => true, :desc => "The input path containing images "
+    
+    method_option :glob, :aliases => '-g',  :desc => 'The glob to use to find files e.g. \'{*.jpg,*.gif,*.png}\' '
+    method_option :recursive, :aliases => '-r', :type => :boolean, :desc => "Scan sub directories of input for images"
+     
+    method_option :attachment_klass, :required => true, :aliases => '-a', :desc => "Ruby Class name of the Attachment e.g Image, Icon"
+    method_option :attach_to_klass, :required => true, :aliases => '-k', :desc => "Ruby Class name attachment belongs to e.g Product, Blog"
+    method_option :attach_to_klass_field, :required => true, :aliases => '-f', :desc => "Attachment belongs to field e.g Product.image, Blog.digital"
+    
+    method_option :attach_to_lookup_field, :required => true, :aliases => '-l', :desc => "The field to use to find the :attach_to_klass record"
+    
+    method_option :split_file_name_on,  :type => :string, :desc => "delimiter to progressivley split filename for lookup", :default => ' '
+    method_option :case_sensitive, :type => :boolean, :desc => "Use case sensitive where clause to find :attach_to_klass"
+    method_option :use_like, :type => :boolean, :desc => "Use :lookup_field LIKE 'string%' instead of :lookup_field = 'string' in where clauses to find :attach_to_klass"
+  
+    method_option :dummy, :aliases => '-d', :type => :boolean, :desc => "Dummy run, do not actually save attachment"
+       
+    method_option :skip_when_assoc, :aliases => '-x', :type => :boolean, :desc => "Do not process if :attach_to_klass already has an attachment"
+    
+    method_option :verbose, :aliases => '-v', :type => :boolean, :desc => "Verbose logging"
+
+    def attach()
+
+      @attachment_path = options[:input]
+      
+      unless(File.exists?(@attachment_path))
+        puts "ERROR: Supplied Path [#{@attachment_path}] not accesible"
+        exit(-1)
+      end
+      
+      require File.expand_path('config/environment.rb')
+      
+      require 'paperclip/attachment_loader'
+            
+      @verbose = options[:verbose]
+
+      puts "Using Field #{options[:attach_to_field]} for lookup"
+       
+      klazz = ModelMapper::class_from_string( options[:attachment_klass] )     
+      raise "Cannot find Attachment Class #{options[:attachment_klass]}" unless klazz
+       
+      attachment_klazz  = ModelMapper::class_from_string( options[:attach_to_klass] )
+      raise "Cannot find Attach to Class #{options[:attach_to_klass]}" unless klazz
+      
+      opts = options.dup
+      
+      opts[:attach_to_klass] = attachment_klazz # Pass in real Ruby class not string class name
+      
+      loader = DataShift::Paperclip::AttachmentLoader.new(klazz, nil, opts)
+   
+      logger.info "Loading attachments from #{@attachment_path}"
+
+      loader.process_from_filesystem(@attachment_path, opts)
+    
+    end
+  end
+  
+end

data/lib/thor/tools.thor

@@ -19,16 +19,36 @@ module Datashift
   
     include DataShift::Logging
       
-    desc "zip", "Create zip of matching digital files" 
+    desc "zip", "Create zip of files" 
 
     method_option :path, :aliases => '-p', :required => true, :desc => "The path to the digital files"
-    method_option :results, :aliases => '-r', :required => true, :desc => "The path to store resulting zip files"
+    method_option :output, :aliases => '-o', :required => true, :desc => "The resulting zip file name"
  
     def zip()
      
       require 'zip/zip'
       require 'zip/zipfilesystem'
 
+      output = options[:output]
+
+      Zip::ZipOutputStream.open(output) do |zos|
+        Dir[File.join(options[:path], '**', '*.*')].each do |p|
+          zos.put_next_entry(File.basename(p))
+          zos.print IO.read(p)
+        end
+      end
+    end
+    
+    desc "zip_matching", "Create zip of matching digital files e.g zip up pdf, jpg and png versions of a file" 
+
+    method_option :path, :aliases => '-p', :required => true, :desc => "The path to the digital files"
+    method_option :results, :aliases => '-r', :required => true, :desc => "The path to store resulting zip files"
+ 
+    def zip_matching()
+     
+      require 'zip/zip'
+      require 'zip/zipfilesystem'
+
       ready_to_zip = {}
       Dir[File.join(options[:path], '**', '*.*')].each do |p|
         next if File.directory? p
@@ -57,6 +77,7 @@ module Datashift
       end
       
     end   
+    
   end
 
 end

data/spec/Gemfile

@@ -20,11 +20,5 @@ end
 
 gem 'spreadsheet'
 
-gem 'rails', '3.2.7'
-gem 'spree', '1.1.3'
-#gem 'spree', '1.1.2'
-
-#gem 'rails', '3.1.3'
-#gem 'spree', '1.0.0'
-
+gem 'rails', '3.2.8'
 

data/spec/csv_exporter_spec.rb

@@ -10,14 +10,14 @@ require File.dirname(__FILE__) + '/spec_helper'
 require 'erb'
 require 'csv_exporter'
 
-describe 'CSV Loader' do
+describe 'CSV Exporter' do
 
   before(:all) do
     
+    # load our test model definitions - Project etc  
+    require ifixture_file('test_model_defs')  
+  
     db_connect( 'test_file' )    # , test_memory, test_mysql
-
-    # load our test model definitions - Project etc
-    require File.join($DataShiftFixturePath, 'test_model_defs')  
    
     # handle migration changes or reset of test DB
     migrate_up

data/spec/csv_loader_spec.rb

@@ -15,7 +15,7 @@ describe 'CSV Loader' do
   before(:all) do
     
     # load our test model definitions - Project etc
-    require File.join($DataShiftFixturePath, 'test_model_defs')  
+    require ifixture_file('test_model_defs')  
        
     db_connect( 'test_file' )    # , test_memory, test_mysql
     migrate_up