rest-ftp-daemon 0.400.0 → 0.410.0.pre.1

data/lib/rest-ftp-daemon/api/root-test.rb

@@ -0,0 +1,69 @@
+require "grape"
+require 'grape-swagger'
+
+module RestFtpDaemon
+  module API
+    class Root < Grape::API
+
+      ### API Documentation
+      add_swagger_documentation hide_documentation_path: true,
+        api_version: Conf.app_ver,
+        doc_version: Conf.app_ver,
+        mount_path: MOUNT_SWAGGER_JSON,
+        info: {
+          title: Conf.app_name,
+          version: Conf.app_ver,
+          description: "API description for #{Conf.app_name} #{Conf.app_ver}",
+          }
+
+
+      namespace :hudson do
+        desc 'Document root'
+        get '/' do
+        end
+      end
+
+      namespace :hudson do
+        desc 'This gets something.',
+          notes: '_test_'
+
+        get '/simple' do
+          { bla: 'something' }
+        end
+      end
+
+      namespace :colorado do
+        desc 'This gets something for URL using - separator.',
+          notes: '_test_'
+
+        get '/simple-test' do
+          { bla: 'something' }
+        end
+
+
+      desc "List all Jobs", http_codes: [
+        { code: 200, message: "Here are the jobs you requested" },
+        ],
+        is_array: true
+      get "/" do
+        begin
+          # Get jobs to display
+          jobs = RestFtpDaemon::JobQueue.instance.jobs
+
+        rescue StandardError => exception
+          log_error "Exception: #{exception.message}"
+          error!({ error: :api_exception, message: exception.message }, 500)
+
+        else
+          status 200
+          present jobs, with: RestFtpDaemon::API::Entities::Job
+
+        end
+      end
+
+
+      end
+
+    end
+  end
+end

data/lib/rest-ftp-daemon/api/root.rb

@@ -1,4 +1,7 @@
 require "grape"
+require 'grape-swagger'
+# require 'grape-swagger/entity'
+# require 'grape-swagger/representable'
 
 module RestFtpDaemon
   module API
@@ -34,11 +37,24 @@ module RestFtpDaemon
 
 
       ### MOUNTPOINTS
-      mount RestFtpDaemon::API::Status => MOUNT_STATUS
-      mount RestFtpDaemon::API::Jobs => MOUNT_JOBS
-      mount RestFtpDaemon::API::Dashbaord => MOUNT_BOARD
-      mount RestFtpDaemon::API::Config => MOUNT_CONFIG
-      mount RestFtpDaemon::API::Debug => MOUNT_DEBUG
+      mount RestFtpDaemon::API::Status      => MOUNT_STATUS
+      mount RestFtpDaemon::API::Jobs        => MOUNT_JOBS
+      mount RestFtpDaemon::API::Dashbaord   => MOUNT_BOARD
+      mount RestFtpDaemon::API::Config      => MOUNT_CONFIG
+      mount RestFtpDaemon::API::Debug       => MOUNT_DEBUG
+
+
+      ### API Documentation
+      add_swagger_documentation hide_documentation_path: true,
+        api_version: Conf.app_ver,
+        doc_version: Conf.app_ver,
+        mount_path: MOUNT_SWAGGER_JSON,
+        info: {
+          title: Conf.app_name,
+          version: Conf.app_ver,
+          description: "API description for #{Conf.app_name} #{Conf.app_ver}",
+          }
+
 
 
       ### INITIALIZATION

data/lib/rest-ftp-daemon/constants.rb

@@ -5,6 +5,8 @@ DEFAULT_FTP_CHUNK       = 1024  # 1 MB
 DEFAULT_PAGE_SIZE       = 50    # 50 lines
 DEFAULT_RETRY_AFTER     = 10    # 10s
 
+TARGET_BLANK             = "_blank"
+
 
 # Internal job constants
 JOB_RANDOM_LEN          = 8
@@ -38,7 +40,10 @@ LOG_INDENT              = "\t"
 
 # Jobs statuses
 JOB_STATUS_PREPARING    = "preparing"
-JOB_STATUS_RUNNING      = "running"
+JOB_STATUS_WORKING      = "working"
+
+JOB_STATUS_TRANSFORMING = "transforming"
+
 JOB_STATUS_CHECKING_SRC = "checking_source"
 JOB_STATUS_CONNECTING   = "remote_connect"
 JOB_STATUS_CHDIR        = "remote_chdir"
@@ -47,12 +52,17 @@ JOB_STATUS_RENAMING     = "renaming"
 JOB_STATUS_PREPARED     = "prepared"
 JOB_STATUS_DISCONNECTING= "remote_disconnect"
 JOB_STATUS_FINISHED     = "finished"
+
 JOB_STATUS_FAILED       = "failed"
 JOB_STATUS_QUEUED       = "queued"
+
 JOB_STYLES = {
   JOB_STATUS_QUEUED      => :active,
   JOB_STATUS_FAILED      => :warning,
   JOB_STATUS_FINISHED    => :success,
+
+  JOB_STATUS_TRANSFORMING   => :info,
+
   JOB_STATUS_UPLOADING   => :info,
   JOB_STATUS_RENAMING    => :info,
   }
@@ -64,6 +74,11 @@ JOB_METHOD_FTPS          = "ftps"
 JOB_METHOD_SFTP          = "sftp"
 JOB_METHOD_FILE          = "file"
 
+# Jobs types
+JOB_TYPE_TRANSFER        = "transfer"
+JOB_TYPE_VIDEO           = "video"
+JOB_TYPE_DUMMY           = "dummy"
+JOB_TYPES                = [JOB_TYPE_TRANSFER, JOB_TYPE_VIDEO, JOB_TYPE_DUMMY]
 
 # Worker statuses
 WORKER_STATUS_STARTING  = "starting"
@@ -83,6 +98,8 @@ WORKER_STYLES = {
 
 
 # API mountpoints
+MOUNT_SWAGGER_JSON        = "/swagger.json"
+MOUNT_SWAGGER_UI          = "/swagger.html"
 MOUNT_JOBS               = "/jobs"
 MOUNT_BOARD              = "/board"
 MOUNT_STATUS             = "/status"

data/lib/rest-ftp-daemon/counters.rb

@@ -11,9 +11,6 @@ module RestFtpDaemon
 
       # Create mutex
       @mutex = Mutex.new
-
-
-      set :system, :started_at, Time.now
     end
 
     def set group, name, value

data/lib/rest-ftp-daemon/exceptions.rb

@@ -6,20 +6,24 @@ module RestFtpDaemon
 
   class MissingPool               < RestFtpDaemonException; end
   class InvalidWorkerNumber       < RestFtpDaemonException; end
-
+  class QueueCantCreateJob        < RestFtpDaemonException; end
   class JobException              < RestFtpDaemonException; end
   class JobTimeout                < RestFtpDaemonException; end
   class JobNotFound               < RestFtpDaemonException; end
-  class JobUnresolvedTokens       < RestFtpDaemonException; end
-  class JobAssertionFailed        < RestFtpDaemonException; end
-  class JobMissingAttribute       < RestFtpDaemonException; end
-  class JobSourceNotFound         < RestFtpDaemonException; end
-  class JobSourceNotReadable      < RestFtpDaemonException; end
-  class JobTargetUnsupported      < RestFtpDaemonException; end
-  class JobTargetUnparseable      < RestFtpDaemonException; end
-  class JobTargetFileExists       < RestFtpDaemonException; end
-  class JobTargetDirectoryError   < RestFtpDaemonException; end
-  class JobTargetPermissionError  < RestFtpDaemonException; end
-  class JobTooManyOpenFiles       < RestFtpDaemonException; end
+
+
+  class AttributeMissing          < RestFtpDaemonException; end
+  class AssertionFailed           < RestFtpDaemonException; end
+  class UnresolvedTokens          < RestFtpDaemonException; end
+  class LocationParseError        < RestFtpDaemonException; end
+  class UnsupportedScheme         < RestFtpDaemonException; end
+
+  class SourceNotSupported        < RestFtpDaemonException; end
+  class SourceNotFound            < RestFtpDaemonException; end
+
+  class TargetNotSupported        < RestFtpDaemonException; end
+  class TargetFileExists          < RestFtpDaemonException; end
+  class TargetDirectoryError      < RestFtpDaemonException; end
+  class TargetPermissionError     < RestFtpDaemonException; end
 
 end

data/lib/rest-ftp-daemon/helpers/api.rb

@@ -14,9 +14,9 @@ module RestFtpDaemon
     end
 
     def get_censored_config
-      config = Conf.to_hash
-      config[:users] = Conf[:users].keys if Conf[:users]
-      config[:endpoints] = Conf[:endpoints].keys if Conf[:endpoints]
+      config              = Conf.to_hash
+      config[:users]      = Conf[:users].keys if Conf[:users]
+      config[:endpoints]  = Conf[:endpoints].keys if Conf[:endpoints]
       config
     end
 

data/lib/rest-ftp-daemon/helpers/common.rb

@@ -17,15 +17,22 @@ module RestFtpDaemon
       rand(36**len).to_s(36)
     end
 
-    def tokenize item
-      return unless item.is_a? String
-      "[#{item}]"
-    end
-
     def dashboard_url filter = ''
       "#{MOUNT_BOARD}/#{filter}"
     end
 
+    def underscore camel_cased_word
+      camel_cased_word.to_s.gsub(/::/, '/').
+        gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+        gsub(/([a-z\d])([A-Z])/,'\1_\2').
+        tr("-", "_").
+        downcase
+    end
+
+    def exception_to_error exception
+      underscore exception.class.name.split('::').last
+    end
+
     # Dates and times: date with time generator
     # def datetime_full datetime
     #   return "-"  if datetime.nil?

data/lib/rest-ftp-daemon/helpers/views.rb

@@ -30,19 +30,49 @@ module RestFtpDaemon
       return  "label-danger"    if runs > 2
     end
 
-    def job_method_label method
-      return if method.nil?
-      klass = case method
-      when JOB_METHOD_FILE
-        "label-primary"
-      when JOB_METHOD_FTP
-        "label-warning"
-      when JOB_METHOD_FTPS
-        "label-success"
+    def location_style uri
+      case uri
+      when URI::FILE
+        "primary"
+      when URI::FTP
+        "warning"
+      when URI::FTPS
+        "success"
+      when URI::SFTP
+        "success"
       else
-        "label-default"
+        "default"
       end
-      "<div class=\"transfer-method label #{klass}\">#{method.upcase}</div>"
+    end
+
+    def job_style job
+      case job.type
+      when JOB_TYPE_TRANSFER
+        icon_klass = "transfer"
+      when JOB_TYPE_VIDEO
+        icon_klass = "facetime-video"
+      when JOB_TYPE_DUMMY
+        icon_klass = "question-sign"
+      else
+        icon_klass = "label-default"
+      end
+    end
+
+    def location_label uri
+      sprintf(
+        '<div class="transfer-type label label-%s">%s</div>',
+        location_style(uri),
+        uri.class.name.split('::').last
+        )
+    end
+
+    def job_type job
+      sprintf(
+          '<span class="glyphicon glyphicon-%s" alt="%s"></span>&nbsp;%s',
+          job_style(job),
+          job.type,
+          job.type
+          )
     end
 
     def datetime_short datetime

data/lib/rest-ftp-daemon/job.rb

@@ -1,18 +1,27 @@
-# Reprensents work to be done along with parameters to process it
+# FIXME: prepare files list ar prepare_common
+# FIXME: scope classes in submodules like Worker::Transfer, Job::Video
+
+# Represents work to be done along with parameters to process it
 require "securerandom"
 
 module RestFtpDaemon
   class Job
-    include BmcDaemonLib::LoggerHelper
     include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
     include CommonHelpers
 
+    # Logging
+    attr_reader :logger
+    include BmcDaemonLib::LoggerHelper
+
     # Class constants
-    FIELDS = [:source, :target, :label, :priority, :pool, :notify, :overwrite, :mkdir, :tempfile]
+    FIELDS = [:type, :source, :target, :label, :priority, :pool, :notify,
+      :overwrite, :mkdir, :tempfile,
+      :video_vc, :video_ac, :video_custom
+      ]
 
     # Class options
-    attr_reader :logger
     attr_accessor :wid
+    attr_accessor :type
 
     attr_reader :id
     attr_reader :error
@@ -39,7 +48,6 @@ module RestFtpDaemon
 
       # Init context
       @id = job_id.to_s
-      # @infos = {}
       #@updated_at = nil
       @started_at = nil
       @finished_at = nil
@@ -47,6 +55,7 @@ module RestFtpDaemon
       @status = nil
       @runs = 0
       @wid = nil
+      @infos = {}
 
       # Prepare configuration
       @config       = Conf[:transfer] || {}
@@ -71,149 +80,95 @@ module RestFtpDaemon
         @pool = DEFAULT_POOL
       end
 
-      # Set job queue, thus reset
-      # reset
+      # Prepare sources/target
+      prepare_source
+      prepare_target
+
+      # Handle exceptions
+      rescue RestFtpDaemon::UnsupportedScheme => exception
+        return oops :started, exception
     end
 
     def reset
-      # Flag current job
+      # Update job status
+      set_status JOB_STATUS_PREPARING
+
+      # Flag current job timestamps
       @queued_at = Time.now
       @updated_at = Time.now
 
-      # Send first notification
-      log_info "Job.initialize notify[queued]}]"
-      client_notify :queued
+      # Job has been prepared, reset infos
+      set_status JOB_STATUS_PREPARED
+      @infos = {}
+      set_info :job, :prepared_at, Time.now
+      set_info_location :source, @source_loc
+      set_info_location :target, @target_loc
 
-      # Update job status
+      # Update job status, send first notification
       set_status JOB_STATUS_QUEUED
       set_error nil
-      @infos = {}
+      client_notify :queued
+      log_info "Job.reset notify[queued]"
     end
 
+    # Process job
     def process
-      log_info "Job.process update_interval[#{JOB_UPDATE_INTERVAL}]"
-
-      # Prepare job
-      begin
-        prepare
+      # Check prerequisites
+      raise RestFtpDaemon::AssertionFailed, "run/source_loc" unless @source_loc
+      raise RestFtpDaemon::AssertionFailed, "run/target_loc" unless @target_loc
 
-      rescue RestFtpDaemon::JobMissingAttribute => exception
-        return oops :started, exception, "missing_attribute"
+      # Notify we start working
+      log_info "Job.process notify [started]"
+      client_notify :started
 
-      rescue RestFtpDaemon::JobUnresolvedTokens => exception
-        return oops :started, exception, "unresolved_tokens"
-
-      rescue RestFtpDaemon::JobTargetUnparseable => exception
-        return oops :started, exception, "target_unparseable"
-
-      rescue RestFtpDaemon::JobTargetUnsupported => exception
-        return oops :started, exception, "target_unsupported"
-
-      rescue RestFtpDaemon::JobAssertionFailed => exception
-        return oops :started, exception, "assertion_failed"
-
-      rescue URI::InvalidURIError => exception
-        return oops :started, exception, "target_invalid"
-
-      else
-        # Prepare done !
-        set_status JOB_STATUS_PREPARED
-        log_info "Job.process notify [started]"
-        client_notify :started
+      # Before work
+      begin
+        log_debug "Job.process before"
+        before
+      rescue RestFtpDaemon::SourceNotSupported => exception
+        return oops :started, exception
+      rescue Net::FTPConnectionError => exception
+        return oops :started, exception, "ftp_connection_error"
+      rescue StandardError => exception
+        return oops :started, exception, "unexpected_error"
       end
 
-      # Process job
+      # Do the hard work
       begin
-        #return oops :ended, Exception.new, "ftp_perm_error"
-        run
-
-      rescue SocketError => exception
-        return oops :ended, exception, "conn_socket_error"
-
-      rescue EOFError => exception
-        return oops :ended, exception, "conn_eof"
-
-      rescue Errno::EHOSTDOWN => exception
-        return oops :ended, exception, "conn_host_is_down"
-
-      rescue Errno::EPIPE=> exception
-        return oops :ended, exception, "conn_broken_pipe"
-
-      rescue Errno::ENETUNREACH => exception
-        return oops :ended, exception, "conn_unreachable"
-
-      rescue Errno::ECONNRESET => exception
-        return oops :ended, exception, "conn_reset_by_peer"
-
-      rescue Errno::ENOTCONN => exception
-        return oops :ended, exception, "conn_failed"
-
-      rescue Errno::ECONNREFUSED => exception
-        return oops :ended, exception, "conn_refused"
-
-      rescue Timeout::Error, Errno::ETIMEDOUT, Net::ReadTimeout => exception
-        return oops :ended, exception, "conn_timed_out"
-
-      rescue OpenSSL::SSL::SSLError => exception
-        return oops :ended, exception, "conn_openssl_error"
-
-      rescue Net::FTPReplyError => exception
-        return oops :ended, exception, "ftp_reply_error"
-
-      rescue Net::FTPTempError => exception
-        return oops :ended, exception, "ftp_temp_error"
-
-      rescue Net::FTPPermError => exception
-        return oops :ended, exception, "ftp_perm_error"
-
-      rescue Net::FTPProtoError => exception
-        return oops :ended, exception, "ftp_proto_error"
-
-      rescue Net::FTPError => exception
-        return oops :ended, exception, "ftp_error"
-
-      rescue Net::SFTP::StatusException => exception
-        return oops :ended, exception, "sftp_exception"
-
-      rescue Net::SSH::HostKeyMismatch => exception
-        return oops :ended, exception, "sftp_key_mismatch"
-
-      rescue Net::SSH::AuthenticationFailed => exception
-        return oops :ended, exception, "sftp_auth_failed"
-
-      rescue Errno::EMFILE => exception
-        return oops :ended, exception, "too_many_open_files"
-
-      rescue Errno::EINVAL => exception
-        return oops :ended, exception, "invalid_argument", true
-
-      rescue Encoding::UndefinedConversionError => exception
-        return oops :ended, exception, "encoding_error", true
-
-      rescue RestFtpDaemon::JobSourceNotFound => exception
-        return oops :ended, exception, "source_not_found"
-
-      rescue RestFtpDaemon::JobSourceNotReadable => exception
-        return oops :ended, exception, "source_not_readable"
+        log_debug "Job.process work"
+        set_status JOB_STATUS_WORKING
+        work
+      rescue StandardError => exception
+        return oops :started, exception, "unexpected_error"
+      end
 
-      rescue RestFtpDaemon::JobTargetFileExists => exception
-        return oops :ended, exception, "target_file_exists"
+      # Finalize all this
+      begin
+        log_debug "Job.process after"
+        after
+      rescue StandardError => exception
+        return oops :started, exception, "unexpected_error"
+      end
 
-      rescue RestFtpDaemon::JobTargetDirectoryError => exception
-        return oops :ended, exception, "target_directory_missing"
+        # All done !
+      set_status JOB_STATUS_FINISHED
+      log_info "JobVideo.process notify [ended]"
+      client_notify :ended
+    end
 
-      rescue RestFtpDaemon::JobTargetPermissionError => exception
-        return oops :ended, exception, "target_permission_error"
+    def before
+    end
+    def work
+    end
+    def after
+    end
 
-      rescue RestFtpDaemon::JobAssertionFailed => exception
-        return oops :ended, exception, "assertion_failed"
+    def source_uri
+      @source_loc.uri
+    end
 
-      else
-        # All done !
-        set_status JOB_STATUS_FINISHED
-        log_info "Job.process notify [ended]"
-        client_notify :ended
-      end
+    def target_uri
+      @target_loc.uri
     end
 
     def weight
@@ -260,156 +215,35 @@ module RestFtpDaemon
 
     def get_info level1, level2
       @mutex.synchronize do
-        @infos || {}
+        # @infos || {}
         @infos[level1][level2] if @infos[level1].is_a? Hash
       end
     end
 
   protected
 
-    def expand_url path
-      URI.parse replace_tokens(path)
+    def alert_common_method_called
+      log_error "Job PLACEHOLDER METHOD CALLED"
     end
 
-    def contains_brackets item
-      /\[.*\]/.match(item)
+    def prepare_source
+      raise RestFtpDaemon::AttributeMissing, "source" unless @source
+      @source_loc = Location.new @source
+      log_info "Job.prepare_source #{@source_loc.uri}"
     end
 
-    def replace_tokens path
-      # Ensure endpoints are not a nil value
-      return path unless @endpoints.is_a? Enumerable
-      vectors = @endpoints.clone
-
-      # Stack RANDOM into tokens
-      vectors["RANDOM"] = SecureRandom.hex(JOB_RANDOM_LEN)
-
-      # Replace endpoints defined in config
-      newpath = path.clone
-      vectors.each do |from, to|
-        next if to.to_s.blank?
-        newpath.gsub! tokenize(from), to
-      end
-
-      # Ensure result does not contain tokens after replacement
-      raise RestFtpDaemon::JobUnresolvedTokens if contains_brackets newpath
-
-      # All OK, return this URL stripping multiple slashes
-      newpath.gsub(/([^:])\/\//, '\1/')
-    end
-
-    def prepare
-      # Init
-      @source_path = nil
-
-      # Prepare flags
-      flag_prepare :mkdir, false
-      flag_prepare :overwrite, false
-      flag_prepare :tempfile, true
-
-      # Update job status
-      set_status JOB_STATUS_PREPARING
-      @runs += 1
-
-      # Prepare source
-      raise RestFtpDaemon::JobMissingAttribute unless @source
-      @source_path = File.expand_path replace_tokens(@source)
-      set_info :source, :path, @source_path
-      set_info :source, :method, JOB_METHOD_FILE
-
-      # Prepare target
-      raise RestFtpDaemon::JobMissingAttribute unless @target
-      target_uri = expand_url @target
-      set_info :target, :uri, target_uri.to_s
-      set_info :target, :host, target_uri.host
-      @target_path = Path.new target_uri.path, true
-
-      # Prepare remote (case would be preferable but too hard to use,
-      # as target could be of a descendent class of URI:XXX and not matching directlry)
-      if target_uri.is_a? URI::FTP
-        log_info "Job.prepare target_method FTP"
-        set_info :target, :method, JOB_METHOD_FTP
-        @remote = RemoteFTP.new target_uri, log_prefix, debug: @config[:debug_ftp]
-
-      elsif (target_uri.is_a? URI::FTPES) || (target_uri.is_a? URI::FTPS)
-        log_info "Job.prepare target_method FTPES"
-        set_info :target, :method, JOB_METHOD_FTPS
-        @remote = RemoteFTP.new target_uri, log_prefix, debug: @config[:debug_ftps], ftpes: true
-
-      elsif target_uri.is_a? URI::SFTP
-        log_info "Job.prepare target_method SFTP"
-        set_info :target, :method, JOB_METHOD_SFTP
-        @remote = RemoteSFTP.new target_uri, log_prefix, debug: @config[:debug_sftp]
-
-      else
-        log_info "Job.prepare unknown scheme [#{target_uri.scheme}]"
-        raise RestFtpDaemon::JobTargetUnsupported
-
-      end
+    def prepare_target
+      raise RestFtpDaemon::AttributeMissing, "target" unless @target
+      @target_loc = Location.new @target
+      log_info "Job.prepare_target #{@target_loc.uri}"
     end
 
-    def run
-      # Update job status
-      set_status JOB_STATUS_RUNNING
-      @started_at = Time.now
-
-      # Method assertions and init
-      raise RestFtpDaemon::JobAssertionFailed, "run/1" unless @source_path
-      raise RestFtpDaemon::JobAssertionFailed, "run/2" unless @target_path
-      @transfer_sent = 0
-      set_info :source, :processed, 0
-
-      # Guess source files from disk
-      set_status JOB_STATUS_CHECKING_SRC
-      sources = find_local @source_path
-      set_info :source, :count, sources.count
-      set_info :source, :files, sources.collect(&:full)
-      log_info "Job.run sources #{sources.collect(&:name)}"
-      raise RestFtpDaemon::JobSourceNotFound if sources.empty?
-
-      # Guess target file name, and fail if present while we matched multiple sources
-      raise RestFtpDaemon::JobTargetDirectoryError if @target_path.name && sources.count>1
-
-      # Connect to remote server and login
-      set_status JOB_STATUS_CONNECTING
-      @remote.connect
-
-      # Prepare target path or build it if asked
-      set_status JOB_STATUS_CHDIR
-      @remote.chdir_or_create @target_path.dir, @mkdir
-
-      # Compute total files size
-      @transfer_total = sources.collect(&:size).sum
-      set_info :transfer, :total, @transfer_total
-
-      # Reset counters
-      @last_data = 0
-      @last_time = Time.now
-
-      # Handle each source file matched, and start a transfer
-      source_processed = 0
-      targets = []
-      sources.each do |source|
-        # Compute target filename
-        full_target = @target_path.clone
-
-        # Add the source file name if none found in the target path
-        unless full_target.name
-          full_target.name = source.name
-        end
-
-        # Do the transfer, for each file
-        remote_push source, full_target
-
-        # Add it to transferred target names
-        targets << full_target.full
-        set_info :target, :files, targets
-
-        # Update counters
-        set_info :source, :processed, source_processed += 1
-      end
-
-      # FTP transfer finished
-      finalize
+    def set_info_location prefix, location
+      return unless location.is_a? Location
+      set_info prefix, :location_uri,    location.to_s
+      set_info prefix, :location_scheme, location.scheme
+      set_info prefix, :location_path,   location.path
+      set_info prefix, :location_host,   location.host
     end
 
   private
@@ -418,7 +252,7 @@ module RestFtpDaemon
      [@wid, @id, nil]
     end
 
-    def find_local path
+    def scan_local_paths path
       Dir.glob(path).collect do |file|
         next unless File.readable? file
         next unless File.file? file
@@ -432,6 +266,20 @@ module RestFtpDaemon
       Thread.current.thread_variable_set :updated_at, now
     end
 
+    def set_error value
+      @mutex.synchronize do
+        @error = value
+      end
+      touch_job
+    end
+
+    def set_status value
+      @mutex.synchronize do
+        @status = value
+      end
+      touch_job
+    end
+
     def set_info level1, level2, value
       @mutex.synchronize do
         @infos || {}
@@ -445,26 +293,14 @@ module RestFtpDaemon
         else
           @infos[level1][level2] = value
         end
-
-        # Mark the job as updated
-        touch_job
       end
+      touch_job
     end
 
     def utf8 value
       value.to_s.encode("UTF-8")
     end
 
-    def set_error value
-      @error = value
-      touch_job
-    end
-
-    def set_status value
-      @status = value
-      touch_job
-    end
-
     def flag_prepare name, default
       # build the flag instance var name
       variable = "@#{name}"
@@ -478,120 +314,6 @@ module RestFtpDaemon
       end
     end
 
-    def finalize
-      # Close FTP connexion and free up memory
-      log_info "Job.finalize"
-      @remote.close
-
-      # Free-up remote object
-      @remote = nil
-
-      # Update job status
-      set_status JOB_STATUS_DISCONNECTING
-      @finished_at = Time.now
-
-      # Update counters
-      RestFtpDaemon::Counters.instance.increment :jobs, :finished
-      RestFtpDaemon::Counters.instance.add :data, :transferred, @transfer_total
-    end
-
-    def remote_push source, target
-      # Method assertions
-      raise RestFtpDaemon::JobAssertionFailed, "remote_push/1" if @remote.nil?
-      raise RestFtpDaemon::JobAssertionFailed, "remote_push/2" if source.nil?
-      raise RestFtpDaemon::JobAssertionFailed, "remote_push/3" if target.nil?
-
-      # Use source filename if target path provided none (typically with multiple sources)
-      log_info "Job.remote_push [#{source.name}]: [#{source.full}] > [#{target.full}]"
-      set_info :source, :current, source.name
-
-      # Compute temp target name
-      tempname = nil
-      if @tempfile
-        tempname = "#{target.name}.temp-#{identifier(JOB_TEMPFILE_LEN)}"
-        log_debug "Job.remote_push tempname [#{tempname}]"
-      end
-
-      # Remove any existing version if expected, or test its presence
-      if @overwrite
-        @remote.remove! target
-      elsif size = @remote.present?(target)
-        log_debug "Job.remote_push existing (#{format_bytes size, 'B'})"
-        raise RestFtpDaemon::JobTargetFileExists
-      end
-
-      # Start transfer
-      transfer_started_at = Time.now
-      @progress_at = 0
-      @notified_at = transfer_started_at
-
-      # Start the transfer, update job status after each block transfer
-      set_status JOB_STATUS_UPLOADING
-      @remote.push source, target, tempname do |transferred, name|
-        # Update transfer statistics
-        progress transferred, name
-
-        # Touch my worker status
-        touch_job
-      end
-
-      # Compute final bitrate
-      global_transfer_bitrate = get_bitrate @transfer_total, (Time.now - transfer_started_at)
-      set_info :transfer, :bitrate, global_transfer_bitrate.round(0)
-
-      # Done
-      set_info :source, :current, nil
-    end
-
-    def progress transferred, name = ""
-      # What's current time ?
-      now = Time.now
-      notify_after = @config[:notify_after]
-
-      # Update counters
-      @transfer_sent += transferred
-      set_info :transfer, :sent, @transfer_sent
-
-      # Update job info
-      percent0 = (100.0 * @transfer_sent / @transfer_total).round(0)
-      set_info :transfer, :progress, percent0
-
-      # Update job status after each NOTIFY_UPADE_STATUS
-      progressed_ago = (now.to_f - @progress_at.to_f)
-      if (!JOB_UPDATE_INTERVAL.to_f.zero?) && (progressed_ago > JOB_UPDATE_INTERVAL.to_f)
-        @current_bitrate = running_bitrate @transfer_sent
-        set_info :transfer, :bitrate, @current_bitrate.round(0)
-
-        # Log progress
-        stack = []
-        stack << "#{percent0} %"
-        stack << (format_bytes @transfer_sent, "B")
-        stack << (format_bytes @transfer_total, "B")
-        stack << (format_bytes @current_bitrate.round(0), "bps")
-        stack2 = stack.map { |txt| ("%#{LOG_PIPE_LEN.to_i}s" % txt) }.join("\t")
-        log_debug "progress #{stack2} \t#{name}"
-
-        # Remember when we last did it
-        @progress_at = now
-      end
-
-      # Notify if requested
-      notified_ago = (now.to_f - @notified_at.to_f)
-      if (!notify_after.nil?) && (notified_ago > notify_after)
-        # Prepare and send notification
-        notif_status = {
-          progress: percent0,
-          transfer_sent: @transfer_sent,
-          transfer_total: @transfer_total,
-          transfer_bitrate: @current_bitrate.round(0),
-          }
-        client_notify :progress, status: notif_status
-
-        # Remember when we last did it
-        @notified_at = now
-      end
-    end
-
     def client_notify event, payload = {}
       # Skip if no URL given
       return unless @notify
@@ -605,34 +327,12 @@ module RestFtpDaemon
       log_error "Job.client_notify EXCEPTION: #{ex.inspect}"
     end
 
-    def get_bitrate delta_data, delta_time
-      return nil if delta_time.nil? || delta_time.zero?
-      8 * delta_data.to_f.to_f / delta_time
-    end
-
-    def running_bitrate current_data
-      return if @last_time.nil?
-
-      # Compute deltas
-      @last_data ||= 0
-      delta_data = current_data - @last_data
-      delta_time = Time.now - @last_time
-
-      # Update counters
-      @last_time = Time.now
-      @last_data = current_data
-
-      # Return bitrate
-      get_bitrate delta_data, delta_time
-    end
-
     def oops event, exception, error = nil, include_backtrace = false
-      # Log this error
-      # error = exception.class.to_s.encoding.to_s if error.nil?
-      error = exception.class if error.nil?
-      # error = "DEF #{exception.class}" if error.nil?
-
+      # Default error code derived from  exception name
+      error = exception_to_error(exception) if error.nil?
       message = "Job.oops event[#{event}] error[#{error}] ex[#{exception.class}] #{exception.message}"
+
+      # Backtrace?
       if include_backtrace
         log_error message, exception.backtrace
       else
@@ -667,8 +367,6 @@ module RestFtpDaemon
     end
 
     # NewRelic instrumentation
-    add_transaction_tracer :prepare,        category: :task
-    add_transaction_tracer :run,            category: :task
     add_transaction_tracer :client_notify,  category: :task
     add_transaction_tracer :initialize,     category: :task