datashift 0.16.0 → 0.40.0

data/lib/tasks/import.thor

@@ -0,0 +1,142 @@
+# Copyright:: (c) Autotelik Media Ltd 2016
+# Author ::   Tom Statter
+# Date ::     Mar 2016
+# License::   MIT.
+#
+require 'thor'
+
+# Note, not DataShift, case sensitive, create namespace for command line : datashift
+module Datashift
+
+  class Import < Thor
+
+    include DataShift::Logging
+
+=begin - These class_option don't seem to currently bet fit for purpose for example
+
+    >>thor help datashift:import:csv
+
+    No value provided for required options '--model', '--input'
+=end
+
+=begin -
+    class_option :model, aliases: '-m', required: true, desc: 'The related active record model'
+
+    class_option :input, aliases: '-i', required: true, desc: 'The input file'
+
+    class_option :loader, aliases: '-l', required: false, desc: 'Loader class to use'
+
+    class_option :verbose, aliases: '-v', type: :boolean, desc: 'Verbose logging'
+
+    class_option :config, aliases: '-c', desc: 'YAML config file with defaults, over-rides etc'
+=end
+
+
+    desc "load", "Import data from file for specific active record model"
+
+    method_option :model, aliases: '-m', required: true, desc: 'The related active record model'
+
+    method_option :input, aliases: '-i', required: true, desc: 'The input file'
+
+    method_option :loader, aliases: '-l', required: false, desc: 'Loader class to use'
+
+    method_option :verbose, aliases: '-v', type: :boolean, desc: 'Verbose logging'
+
+    method_option :config, aliases: '-c', desc: 'YAML config file with defaults, over-rides etc'
+
+    def load()
+      start_connections
+
+      importer = if(options[:loader])
+                   logger.info("Attempting to use supplied Loader : #{options[:loader]}")
+                   DataShift::MapperUtils::class_from_string(options[:loader]).new
+                 else
+                   logger.info("No Loader specified - finding appropriate Loader for file type")
+                   DataShift::Loader::Factory.get_loader(options[:input])
+                 end
+
+      import(importer)
+    end
+
+
+    desc "excel", "Import .xls file for specifiec active record model"
+
+    method_option :model, aliases: '-m', required: true, desc: 'The related active record model'
+
+    method_option :input, aliases: '-i', required: true, desc: 'The input file'
+
+    method_option :loader, aliases: '-l', required: false, desc: 'Loader class to use'
+
+    method_option :verbose, aliases: '-v', type: :boolean, desc: 'Verbose logging'
+
+    method_option :config, aliases: '-c', desc: 'YAML config file with defaults, over-rides etc'
+
+    def excel()
+
+      start_connections
+
+      importer = if(options[:loader])
+                   logger.info("Attempting to use supplied Loader : #{options[:loader]}")
+                   DataShift::MapperUtils::class_from_string(options[:loader]).new
+                 else
+                   logger.info("No Loader specified - using standard Excel Loader")
+                   DataShift::ExcelLoader.new
+                 end
+
+      import(importer)
+    end
+
+    desc "csv", "Import CSV file for specified active record model"
+
+    method_option :model, aliases: '-m', required: true, desc: 'The related active record model'
+
+    method_option :input, aliases: '-i', required: true, desc: 'The input file'
+
+    method_option :loader, aliases: '-l', required: false, desc: 'Loader class to use'
+
+    method_option :verbose, aliases: '-v', type: :boolean, desc: 'Verbose logging'
+
+    method_option :config, aliases: '-c', desc: 'YAML config file with defaults, over-rides etc'
+
+    def csv()
+
+      start_connections
+
+      importer = if(options[:loader])
+                   logger.info("Attempting to use supplied Loader : #{options[:loader]}")
+                   DataShift::MapperUtils::class_from_string(options[:loader]).new
+                 else
+                   logger.info("No Loader specified - using standard Csv Loader")
+                   DataShift::CsvLoader.new
+                 end
+
+      import(importer)
+    end
+
+    no_commands do
+      def start_connections
+
+        if File.exist?(File.expand_path('config/environment.rb'))
+          begin
+            require File.expand_path('config/environment.rb')
+          rescue => e
+            logger.error("Failed to initialise ActiveRecord : #{e.message}")
+            raise ConnectionError.new("Failed to initialise ActiveRecord : #{e.message}")
+          end
+
+        else
+          raise PathError.new('No config/environment.rb found - cannot initialise ActiveRecord')
+        end
+      end
+
+      def import(importer)
+        logger.info "Datashift: Starting Import from #{options[:input]}"
+
+        importer.configure_from( options[:config] ) if(options[:config])
+
+        importer.run(options[:input], options[:model])
+      end
+    end
+
+  end
+end

data/lib/{thor → tasks}/paperclip.thor

@@ -1,39 +1,31 @@
-# Copyright:: (c) Autotelik Media Ltd 2012
+# Copyright:: (c) Autotelik Media Ltd 2016
 # Author ::   Tom Statter
-# Date ::     Sept 2012
+# Date ::     March 2016
 # License::   MIT.
 #
 # Usage::
 #
-#  To pull Datashift commands into your main application :
-#
-#     require 'datashift'
-#
-#     DataShift::load_commands
-#
 #     thor help datashift:paperclip:attach
 #
-require 'datashift'
+require 'thor'
 
 # Note, not DataShift, case sensitive, create namespace for command line : datashift
 module Datashift
-         
-  class Paperclip < Thor     
-  
-    include DataShift::Logging   
+
+  class Paperclip < Thor
 
     desc "attach", "Create paperclip attachments and attach to a Model from files in a directory.
     This is specifically for the use case where the paperclip attachments are stored in a class, such as Image, Icon, Asset,
     and this class has a relationship, such as belongs_to, with another class, such as Product, User, Document.
- 
+
     Each matching file is used to create an instance of the paperclip attachment, given by :attachment_klass.
-    
+
     The class with the relationship, can be specified via :attach_to_klass
 
-      Examples     
+      Examples
         Owner has_many pdfs and mp3 files as Digitals .... :attachment_klass = Digital and :attach_to_klass = Owner
         User has a single Image used as an avatar ... attachment_klass = Image and :attach_to_klass = User
-    
+
     The file name is used to lookup the instance of :attach_to_klass to assign the new attachment to, via :attach_to_find_by_field
 
     So say we have a file called smithj_avatar.gif, and we want to lookup Users by login
@@ -42,83 +34,91 @@ module Datashift
 
     Once instance of :attach_to_klass found, the new attachment is assigned.
 
-     The attribute to assign new attachment to is gtiven by :attach_to_field    
-  
+     The attribute to assign new attachment to is gtiven by :attach_to_field
+
      Examples
         :attach_to_field => digitals  : Owner.digitals = attachment(Digital)
         :attach_to_field => avatar    : User.avatar = attachment(Image)"
-    
+
 
     # :dummy => dummy run without actual saving to DB
     method_option :input, :aliases => '-i', :required => true, :desc => "The input path containing images "
-    
+
     method_option :glob, :aliases => '-g',  :desc => 'The glob to use to find files e.g. \'{*.jpg,*.gif,*.png}\' '
     method_option :recursive, :aliases => '-r', :type => :boolean, :desc => "Scan sub directories of input for images"
-     
+
     method_option :attachment_klass, :required => true, :aliases => '-a', :desc => "Ruby Class name of the Attachment e.g Image, Icon"
-    
-    method_option :attach_to_klass, :required => true, :aliases => '-k', :desc => "A class that has a relationship with the attachment (has_many, has_one, belongs_to)"     
+
+    method_option :attach_to_klass, :required => true, :aliases => '-k', :desc => "A class that has a relationship with the attachment (has_many, has_one, belongs_to)"
     method_option :attach_to_find_by_field, :required => true, :aliases => '-l', :desc => "The field to use to find the :attach_to_klass record"
     method_option :attach_to_field, :required => true, :aliases => '-f', :desc => "Attachment belongs to field e.g Product.image, Blog.digital"
-  
-    
-      # => :attach_to_find_by_field    
-      #       For the :attach_to_klass, this is the field used to search for the parent
-      #       object to assign the new attachment to.
-      #     Examples     
-      #       Owner has a unique 'name' field ... :attach_to_find_by_field = :name
-      #       User has a unique  'login' field  ... :attach_to_klass = :login
-      #
-      # => :attach_to_field    
-      #       Attribute/association to assign attachment to on :attach_to_klass.
-      #      Examples
-      #         :attach_to_field => digitals  : Owner.digitals = attachment
-      #         :attach_to_field => avatar    : User.avatar = attachment
-      
-    method_option :split_file_name_on,  :type => :string, :desc => "delimiter to progressivley split filename for lookup", :default => ' '
+
+
+    # => :attach_to_find_by_field
+    #       For the :attach_to_klass, this is the field used to search for the parent
+    #       object to assign the new attachment to.
+    #     Examples
+    #       Owner has a unique 'name' field ... :attach_to_find_by_field = :name
+    #       User has a unique  'login' field  ... :attach_to_klass = :login
+    #
+    # => :attach_to_field
+    #       Attribute/association to assign attachment to on :attach_to_klass.
+    #      Examples
+    #         :attach_to_field => digitals  : Owner.digitals = attachment
+    #         :attach_to_field => avatar    : User.avatar = attachment
+
+    method_option :split_file_name_on,  :type => :string,
+                  :desc => "delimiter to progressivley split file_name for lookup", :default => ' '
+
     method_option :case_sensitive, :type => :boolean, :desc => "Use case sensitive where clause to find :attach_to_klass"
     method_option :use_like, :type => :boolean, :desc => "Use :lookup_field LIKE 'string%' instead of :lookup_field = 'string' in where clauses to find :attach_to_klass"
-  
-    method_option :dummy, :aliases => '-d', :type => :boolean, :desc => "Dummy run, do not actually save attachment"
-       
+
     method_option :skip_when_assoc, :aliases => '-x', :type => :boolean, :desc => "Do not process if :attach_to_klass already has an attachment"
-    
+
     method_option :verbose, :aliases => '-v', :type => :boolean, :desc => "Verbose logging"
 
     def attach()
 
       @attachment_path = options[:input]
-      
-      unless(File.exists?(@attachment_path))
+
+      unless(File.exist?(@attachment_path))
         puts "ERROR: Supplied Path [#{@attachment_path}] not accesible"
         exit(-1)
       end
-      
-      require File.expand_path('config/environment.rb')
-      
+
+      start_connections
+
       require 'paperclip/attachment_loader'
-            
-      @verbose = options[:verbose]
-
-      puts "Using Field #{options[:attach_to_field]} for lookup"
-       
-      klazz = ModelMapper::class_from_string( options[:attachment_klass] )     
-      raise "Cannot find Attachment Class #{options[:attachment_klass]}" unless klazz
-       
-      attachment_klazz  = ModelMapper::class_from_string( options[:attach_to_klass] )
-      raise "Cannot find Attach to Class #{options[:attach_to_klass]}" unless klazz
-      
-      opts = options.dup
-      
-      opts[:attach_to_klass] = attachment_klazz # Pass in real Ruby class not string class name
-      
-      loader = DataShift::Paperclip::AttachmentLoader.new(klazz, nil, opts)
-   
+
+      puts "Using Field #{options[:attach_to_find_by_field]} to lookup matching [#{options[:attach_to_klass]}]"
+
+      loader = DataShift::Paperclip::AttachmentLoader.new
+
+      loader.init_from_options(options)
+
       logger.info "Loading attachments from #{@attachment_path}"
 
-      loader.process_from_filesystem(@attachment_path, opts)
-    
+      loader.run(@attachment_path, options[:attachment_klass])
+    end
+
+    no_commands do
+
+      def start_connections
+
+        if File.exist?(File.expand_path('config/environment.rb'))
+          begin
+            require File.expand_path('config/environment.rb')
+          rescue => e
+            logger.error("Failed to initialise ActiveRecord : #{e.message}")
+            raise ConnectionError.new("Failed to initialise ActiveRecord : #{e.message}")
+          end
+
+        else
+          raise PathError.new('No config/environment.rb found - cannot initialise ActiveRecord')
+        end
+      end
+
     end
   end
-  
+
 end

data/{tasks → lib/tasks/to_convert_to_thor}/db_tasks.rake

@@ -1,45 +1,45 @@
-namespace :datashift  do
-  
-  namespace :db  do
+namespace :datashift do
 
-    SYSTEM_TABLE_EXCLUSION_LIST = ['schema_migrations']
+  namespace :db do
 
-    desc "Purge the current database"
-    task :purge, [:exclude_system_tables] => [:environment] do |t, args|
+    SYSTEM_TABLE_EXCLUSION_LIST = ['schema_migrations'].freeze
+
+    desc 'Purge the current database'
+    task :purge, [:exclude_system_tables] => [:environment] do |_t, args|
       require 'highline/import'
 
-      if(Rails.env.production?)
-        agree("WARNING: In Production database, REALLY PURGE ? [y]:")
+      if Rails.env.production?
+        agree('WARNING: In Production database, REALLY PURGE ? [y]:')
       end
 
       config = ActiveRecord::Base.configurations[Rails.env || 'development']
       case config['adapter']
-      when "mysql", "mysql2", "jdbcmysql"
+      when 'mysql', 'mysql2', 'jdbcmysql'
         ActiveRecord::Base.establish_connection(config)
         ActiveRecord::Base.connection.tables.each do |table|
-          next if(args[:exclude_system_tables] && SYSTEM_TABLE_EXCLUSION_LIST.include?(table) )
+          next if args[:exclude_system_tables] && SYSTEM_TABLE_EXCLUSION_LIST.include?(table)
           puts "purging table: #{table}"
           ActiveRecord::Base.connection.execute("TRUNCATE #{table}")
         end
-      when "sqlite","sqlite3"
-        dbfile = config["database"] || config["dbfile"]
+      when 'sqlite', 'sqlite3'
+        dbfile = config['database'] || config['dbfile']
         File.delete(dbfile) if File.exist?(dbfile)
-      when "sqlserver"
-        dropfkscript = "#{config["host"]}.#{config["database"]}.DP1".gsub(/\\/,'-')
-        `osql -E -S #{config["host"]} -d #{config["database"]} -i db\\#{dropfkscript}`
-        `osql -E -S #{config["host"]} -d #{config["database"]} -i db\\#{Rails.env}_structure.sql`
-      when "oci", "oracle"
+      when 'sqlserver'
+        dropfkscript = "#{config['host']}.#{config['database']}.DP1".tr('\\', '-')
+        `osql -E -S #{config['host']} -d #{config['database']} -i db\\#{dropfkscript}`
+        `osql -E -S #{config['host']} -d #{config['database']} -i db\\#{Rails.env}_structure.sql`
+      when 'oci', 'oracle'
         ActiveRecord::Base.establish_connection(config)
         ActiveRecord::Base.connection.structure_drop.split(";\n\n").each do |ddl|
           ActiveRecord::Base.connection.execute(ddl)
         end
-      when "firebird"
+      when 'firebird'
         ActiveRecord::Base.establish_connection(config)
         ActiveRecord::Base.connection.recreate_database!
       else
-        raise "Task not supported by '#{config["adapter"]}'"
+        raise "Task not supported by '#{config['adapter']}'"
       end
     end
 
   end
-end
+end

data/lib/tasks/tools.thor

@@ -0,0 +1,109 @@
+# Copyright:: (c) Autotelik Media Ltd 2016
+# Author ::   Tom Statter
+# Date ::     April 2016
+# License::   MIT.
+#
+#
+require 'datashift'
+  
+# Note, not DataShift, case sensitive, create namespace for command line : datashift
+module Datashift
+
+  class Tools < Thor     
+  
+    include DataShift::Logging
+
+    desc 'file_rename', 'Clone a folder of files, consistently renaming each in the process'
+
+    method_option :path, :aliases => '-p', required: true, desc: "The path to the original files"
+    method_option :output, :aliases => '-o', required: true, desc: "The resulting zip file name"
+    method_option :offset,  required: false, type: :numeric, desc: "A numeric offset to add to the file name"
+    method_option :width,  required: false, type: :numeric, desc: "A numeric width tp pad the file name"
+    method_option :prefix,  required: false, desc: "A strign prefix to add to file name"
+    method_option :commit,  required: false, type: :boolean, desc: "Actually perform copy"
+
+    def file_rename
+
+      cache = options[:path]
+
+      if File.exist?(cache)
+        puts "Renaming files from #{cache}"
+        Dir.glob(File.join(cache, '*')) do |name|
+          path, base_name = File.split(name)
+          id = base_name.slice!(/\w+/)
+
+          id = id.to_i + options[:offset].to_i if options[:offset]
+          id = "%0#{width}d" % id.to_i if options[:width]
+          id = options[:prefix] + id.to_s if options[:prefix]
+
+          destination = File.join( options[:output], "#{id}#{base_name}")
+          puts "File Rename: cp #{name} #{destination}"
+
+          File.send( 'cp', name, destination) if options[:commit]
+        end
+      end
+    end
+
+    desc "zip", "Create zip of files" 
+
+    method_option :path, :aliases => '-p', required: true, desc: "The path to the digital files"
+    method_option :output, :aliases => '-o', required: true, desc: "The resulting zip file name"
+ 
+    def zip
+     
+      require 'zip/zip'
+      require 'zip/zipfilesystem'
+
+      output = options[:output]
+
+      Zip::ZipOutputStream.open(output) do |zos|
+        Dir[File.join(options[:path], '**', '*.*')].each do |p|
+          zos.put_next_entry(File.basename(p))
+          zos.print IO.read(p)
+        end
+      end
+    end
+    
+    desc "zip_matching", "Create zip of matching digital files e.g zip up pdf, jpg and png versions of a file" 
+
+    method_option :path, :aliases => '-p', required: true, desc: "The path to the digital files"
+    method_option :results, :aliases => '-r', required: true, desc: "The path to store resulting zip files"
+ 
+    def zip_matching()
+     
+      require 'zip/zip'
+      require 'zip/zipfilesystem'
+
+      ready_to_zip = {}
+      Dir[File.join(options[:path], '**', '*.*')].each do |p|
+        next if File.directory? p
+        
+        basename = File.basename(p, '.*')
+        ready_to_zip[basename] ||= []       
+        ready_to_zip[basename] << p  
+      end 
+      
+      output = options[:results]
+      
+      FileUtils::mkdir_p(output) unless File.exist?(output)
+      
+      puts "Creating #{ready_to_zip.keys.size} new zips"
+      ready_to_zip.each do |basename, paths|
+      
+        z= File.join(output, basename + '.zip')
+        puts "zipping to #{z}"
+        
+        Zip::ZipOutputStream.open(z) do |zos|
+          paths.each do |file|
+            zos.put_next_entry(File.basename(file))
+            zos.print IO.read(file)
+          end
+        end
+      end
+      
+    end   
+    
+  end
+
+end
+