rest-ftp-daemon 0.222.0 → 0.230.0

data/Rakefile

@@ -7,4 +7,10 @@ RSpec::Core::RakeTask.new(:spec)
 
 # Run specs by default
 desc 'Run all tests'
-task :default => :spec
+
+require 'rubocop/rake_task'
+RuboCop::RakeTask.new(:rubocop) do |task|
+  task.fail_on_error = false
+end
+
+task :default => [:spec, :rubocop]

data/bin/rest-ftp-daemon

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
 # Try to load external libs
-app_root = File.dirname(__FILE__) + "/../"
+app_root = File.expand_path File.dirname(__FILE__) + "/../"
 begin
   require "thin"
   require "optparse"
@@ -20,6 +20,18 @@ end
 puts
 
 
+# Provide default config file information
+DEFAULT_CONFIG_PATH = File.expand_path "/etc/#{APP_NAME}.yml"
+SAMPLE_CONFIG_FILE = File.expand_path("#{app_root}/rest-ftp-daemon.yml.sample")
+#SAMPLE_CONFIG_FILE = File.expand_path("#{app_root}/#{APP_NAME}.yml.sample")
+TAIL_MESSAGE = <<EOD
+
+A default configuration is available here: #{SAMPLE_CONFIG_FILE}.
+You should copy it to the expected location #{DEFAULT_CONFIG_PATH}:
+
+sudo cp #{SAMPLE_CONFIG_FILE} #{DEFAULT_CONFIG_PATH}
+EOD
+
 # Detect options from ARGV
 options = {}
 parser = OptionParser.new do |opts|
@@ -35,11 +47,17 @@ parser = OptionParser.new do |opts|
   opts.on("-u", "--user NAME", "User to run daemon as (use with -g)")  { |user| options["user"] = user }
   opts.on("-g", "--group NAME", "Group to run daemon as (use with -u)"){ |group| options["group"] = group }
 
-  opts.on_tail("-h", "--help", "Show this message")                    { puts opts; exit }
+  opts.separator ""
+  opts.on_tail("-h", "--help", "Show this message")                    do
+    puts opts
+    puts TAIL_MESSAGE unless File.exists?(DEFAULT_CONFIG_PATH)
+    exit
+  end
   opts.on_tail("-v", "--version", "Show version (#{APP_VER})")         { puts APP_VER; exit }
 end
 
 
+
 # Parse options and check compliance
 begin
   parser.order!(ARGV)
@@ -52,7 +70,8 @@ end
 
 
 # Load config, and merge options from ARGV into settings
-APP_CONF ||= File.expand_path "/etc/#{APP_NAME}.yml"
+# FIXME: file configuration detection could reside in settings.rb
+APP_CONF ||= DEFAULT_CONFIG_PATH
 abort "EXITING: cannot read configuration file: #{APP_CONF}" unless File.exists? APP_CONF
 begin
   # Import settings

data/lib/rest-ftp-daemon.rb

@@ -12,6 +12,7 @@ require "timeout"
 require "sys/cpu"
 require "syslog"
 require "net/ftp"
+require "net/sftp"
 require "net/http"
 require "double_bag_ftps"
 require "thread"
@@ -24,26 +25,29 @@ require "get_process_mem"
 
 
 # Project's libs
-require "rest-ftp-daemon/constants"
-require "rest-ftp-daemon/settings"
-require "rest-ftp-daemon/exceptions"
-require "rest-ftp-daemon/helpers"
-require "rest-ftp-daemon/logger_helper"
-require "rest-ftp-daemon/logger_pool"
-require "rest-ftp-daemon/logger"
-require "rest-ftp-daemon/paginate"
-require "rest-ftp-daemon/uri"
-require "rest-ftp-daemon/job_queue"
-require "rest-ftp-daemon/worker"
-require "rest-ftp-daemon/worker_conchita"
-require "rest-ftp-daemon/worker_job"
-require "rest-ftp-daemon/worker_pool"
-require "rest-ftp-daemon/job"
-require "rest-ftp-daemon/notification"
+require_relative "rest-ftp-daemon/constants"
+require_relative "rest-ftp-daemon/settings"
+require_relative "rest-ftp-daemon/exceptions"
+require_relative "rest-ftp-daemon/helpers"
+require_relative "rest-ftp-daemon/logger_helper"
+require_relative "rest-ftp-daemon/logger_pool"
+require_relative "rest-ftp-daemon/logger"
+require_relative "rest-ftp-daemon/paginate"
+require_relative "rest-ftp-daemon/uri"
+require_relative "rest-ftp-daemon/job_queue"
+require_relative "rest-ftp-daemon/worker"
+require_relative "rest-ftp-daemon/worker_conchita"
+require_relative "rest-ftp-daemon/worker_job"
+require_relative "rest-ftp-daemon/worker_pool"
+require_relative "rest-ftp-daemon/job"
+require_relative "rest-ftp-daemon/notification"
 
-require "rest-ftp-daemon/api/job_presenter"
-require "rest-ftp-daemon/api/jobs"
-require "rest-ftp-daemon/api/dashboard"
-
-require "rest-ftp-daemon/api/root"
+require_relative "rest-ftp-daemon/path"
+require_relative "rest-ftp-daemon/remote"
+require_relative "rest-ftp-daemon/remote_ftp"
+require_relative "rest-ftp-daemon/remote_sftp"
 
+require_relative "rest-ftp-daemon/api/job_presenter"
+require_relative "rest-ftp-daemon/api/jobs"
+require_relative "rest-ftp-daemon/api/dashboard"
+require_relative "rest-ftp-daemon/api/root"

data/lib/rest-ftp-daemon/constants.rb

@@ -1,19 +1,29 @@
 # Terrific constants
 APP_NAME = "rest-ftp-daemon"
 APP_NICK = "rftpd"
-APP_VER = "0.222.0"
+APP_VER = "0.230.0"
 
 
 # Jobs and workers
 JOB_RANDOM_LEN          = 8
 JOB_IDENT_LEN           = 4
 JOB_TEMPFILE_LEN        = 8
+JOB_UPDATE_INTERVAL     = 1
+
 JOB_STATUS_UPLOADING    = :uploading
 JOB_STATUS_RENAMING     = :renaming
+JOB_STATUS_PREPARED     = :prepared
 JOB_STATUS_FINISHED     = :finished
 JOB_STATUS_FAILED       = :failed
 JOB_STATUS_QUEUED       = :queued
 
+WORKER_STATUS_STARTING  = :starting
+WORKER_STATUS_WAITING   = :waiting
+WORKER_STATUS_RUNNING   = :running
+WORKER_STATUS_FINISHED  = :finished
+WORKER_STATUS_TIMEOUT   = :timeout
+WORKER_STATUS_CRASHED   = :crashed
+WORKER_STATUS_CLEANING  = :cleaning
 
 # Logging and startup
 LOG_PIPE_LEN            = 10
@@ -28,6 +38,10 @@ LOG_FORMAT_PREFIX       = "%s %s\t%-#{LOG_PIPE_LEN.to_i}s\t"
 LOG_FORMAT_MESSAGE      = "%#{-LOG_COL_WID.to_i}s\t%#{-LOG_COL_JID.to_i}s\t%#{-LOG_COL_ID.to_i}s"
 LOG_NEWLINE             = "\n"
 
+LOG_INDENT             = "\t"
+
+
+
 # Notifications
 NOTIFY_PREFIX           = "rftpd"
 NOTIFY_USERAGENT        = "#{APP_NAME} - #{APP_VER}"
@@ -35,26 +49,26 @@ NOTIFY_IDENTIFIER_LEN   = 4
 
 
 # Dashboard row styles
-JOB_STYLES = {
+DASHBOARD_JOB_STYLES = {
   JOB_STATUS_QUEUED     => :active,
   JOB_STATUS_FAILED     => :warning,
   JOB_STATUS_FINISHED   => :success,
   JOB_STATUS_UPLOADING  => :info,
   JOB_STATUS_RENAMING   => :info,
   }
-WORKER_STYLES = {
+DASHBOARD_WORKER_STYLES = {
   waiting:              :success,
   working:              :info,
   crashed:              :danger,
   done:                 :success,
   dead:                 :danger
   }
-PAGINATE_MAX            = 30
 
 
 # Configuration defaults
 DEFAULT_WORKER_TIMEOUT  = 3600
-DEFAULT_FTP_CHUNK       = 2048
+DEFAULT_FTP_CHUNK       = 512
+DEFAULT_PAGE_SIZE       = 40
 
 
 # Initialize defaults

data/lib/rest-ftp-daemon/exceptions.rb

@@ -19,7 +19,8 @@ module RestFtpDaemon
   class JobTargetUnsupported      < RestFtpDaemonException; end
   class JobTargetUnparseable      < RestFtpDaemonException; end
   class JobTargetFileExists       < RestFtpDaemonException; end
-  class JobTargetShouldBeDirectory< RestFtpDaemonException; end
+  class JobTargetDirectoryError   < RestFtpDaemonException; end
+  class JobTargetPermissionError  < RestFtpDaemonException; end
   class JobTooManyOpenFiles       < RestFtpDaemonException; end
 
 end

data/lib/rest-ftp-daemon/helpers.rb

@@ -31,29 +31,33 @@ module RestFtpDaemon
     end
 
     def self.tokenize item
+      return unless item.is_a? String
       "[#{item}]"
     end
 
     def self.highlight_tokens path
+      return unless path.is_a? String
       path.gsub(/(\[[^\[]+\])/, '<span class="token">\1</span>')
     end
 
     def self.extract_filename path
+      return unless path.is_a? String
       # match everything that's after a slash at the end of the string
       m = path.match /\/?([^\/]+)$/
       return m[1] unless m.nil?
     end
 
     def self.extract_dirname path
+      return unless path.is_a? String
       # match all the beginning of the string up to the last slash
       m = path.match(/^(.*)\/[^\/]*$/)
       return "/#{m[1]}" unless m.nil?
     end
 
     def self.extract_parent path
-      # m = path.match(/^(.*\/)[^\/]*\/+$/)
-      m = path.match(/^(.*\/)[^\/]+\/?$/)
-      return m[1] unless m.nil?
+      return unless path.is_a? String
+      m = path.match(/^(.*)\/([^\/]+)\/?$/)
+      return m[1], m[2] unless m.nil?
     end
 
     def self.local_port_used? port
@@ -106,8 +110,9 @@ module RestFtpDaemon
       datetime.to_datetime.strftime("%d/%m %H:%M:%S")
     end
 
-    def self.hide_credentials_from_url url
-      url.sub(/([a-z]+:\/\/[^\/]+):[^\/]+\@/, '\1@' )
+    def self.hide_credentials_from_url path
+      return unless path.is_a? String
+      path.sub(/([a-z]+:\/\/[^\/]+):[^\/]+\@/, '\1@' )
     end
 
   end

data/lib/rest-ftp-daemon/job.rb

@@ -57,24 +57,22 @@ module RestFtpDaemon
       flag_default :tempfile, false
 
       # Read source file size and parameters
-      @ftp_debug_enabled = (Settings.at :debug, :ftp) == true
-      update_every_kb = (Settings.transfer.update_every_kb rescue nil) || DEFAULT_FTP_CHUNK
       @notify_after_sec = Settings.transfer.notify_after_sec rescue nil
-      @chunk_size = update_every_kb * 1024
+      @chunk_size = DEFAULT_FTP_CHUNK * 1024
 
       # Flag current job
       @queued_at = Time.now
       @updated_at = Time.now
 
       # Send first notification
-      log_info "Job.initialize notify[queued] notify_after_sec[#{@notify_after_sec}] update_every_kb[#{@update_every_kb}]"
+      log_info "Job.initialize notify[queued] notify_after_sec[#{@notify_after_sec}] JOB_UPDATE_INTERVAL[#{JOB_UPDATE_INTERVAL}]"
       client_notify :queued
     end
 
     def process
       # Update job's status
       @error = nil
-      log_info "Job.process starting"
+      log_info "Job.process"
 
       # Prepare job
       begin
@@ -99,20 +97,10 @@ module RestFtpDaemon
       rescue RestFtpDaemon::JobAssertionFailed => exception
         return oops :started, exception, :assertion_failed
 
-      # rescue RestFtpDaemon::JobTimeout => exception
-      #   info "Job.process propagate JobTimeout to Worker"
-      #   raise RestFtpDaemon::JobTimeout
-
-      # rescue RestFtpDaemon::RestFtpDaemonException => exception
-      #   return oops :started, exception, :prepare_failed, true
-
-      # rescue StandardError => exception
-      #   return oops :started, exception, :prepare_unhandled, true
-
       else
         # Prepare done !
-        newstatus :prepared
-        log_info "Job.process notify[started]"
+        newstatus JOB_STATUS_PREPARED
+        log_info "Job.process notify [started]"
         client_notify :started
       end
 
@@ -154,6 +142,9 @@ module RestFtpDaemon
       rescue Net::FTPTempError => exception
         return oops :ended, exception, :net_temp_error
 
+      rescue Net::SFTP::StatusException => exception
+        return oops :ended, exception, :sftp_exception
+
       rescue Errno::EMFILE => exception
         return oops :ended, exception, :too_many_open_files
 
@@ -169,26 +160,19 @@ module RestFtpDaemon
       rescue RestFtpDaemon::JobTargetFileExists => exception
         return oops :ended, exception, :target_file_exists
 
-      rescue RestFtpDaemon::JobTargetShouldBeDirectory => exception
-        return oops :ended, exception, :target_not_directory
+      rescue RestFtpDaemon::JobTargetDirectoryError => exception
+        return oops :ended, exception, :target_directory_missing
+
+      rescue RestFtpDaemon::JobTargetPermissionError => exception
+        return oops :ended, exception, :target_permission_error
 
       rescue RestFtpDaemon::JobAssertionFailed => exception
         return oops :ended, exception, :assertion_failed
 
-      # rescue RestFtpDaemon::JobTimeout => exception
-      #   info "Job.process propagate JobTimeout to Worker"
-      #   raise RestFtpDaemon::JobTimeout
-
-      # rescue RestFtpDaemon::RestFtpDaemonException => exception
-      #   return oops :ended, exception, :transfer_failed, true
-
-      # rescue StandardError => exception
-      #   return oops :ended, exception, :transfer_unhandled, true
-
       else
         # All done !
         newstatus JOB_STATUS_FINISHED
-        log_info "Job.process notify[ended]"
+        log_info "Job.process notify [ended]"
         client_notify :ended
       end
 
@@ -274,103 +258,115 @@ module RestFtpDaemon
 
     def prepare
       # Update job status
-      newstatus :preparing
+      newstatus :prepare
 
       # Init
       @source_method = :file
       @target_method = nil
       @source_path = nil
-      @target_url = nil
 
-      # Check source
+      # Prepare source
       raise RestFtpDaemon::JobMissingAttribute unless @source
       @source_path = expand_path @source
       set :source_path, @source_path
       set :source_method, :file
 
-      # Check target
+      # Prepare target
       raise RestFtpDaemon::JobMissingAttribute unless @target
-      @target_url = expand_url @target
-      set :target_url, @target_url.to_s
-
-      if @target_url.is_a? URI::FTP
-        @target_method = :ftp
-      elsif @target_url.is_a? URI::FTPES
-        @target_method = :ftps
-      elsif @target_url.is_a? URI::FTPS
-        @target_method = :ftps
-      end
-      set :target_method, @target_method
+      target_uri = expand_url @target
+      set :target_uri, target_uri.to_s
+      @target_path = Path.new target_uri.path, true
+
+      #puts "@target_path: #{@target_path.inspect}"
+
+      # Prepare remote
+      newstatus :remote_init
+      #FIXME: use a "case" statement on @target_url.class
+
+      if target_uri.is_a? URI::FTP
+        log_info "Job.prepare target_method FTP"
+        set :target_method, :ftp
+        @remote = RemoteFTP.new target_uri, log_context
+
+      elsif (target_uri.is_a? URI::FTPES) || (target_uri.is_a? URI::FTPS)
+        log_info "Job.prepare target_method FTPES"
+        set :target_method, :ftpes
+        @remote = RemoteFTP.new target_uri, log_context, ftpes: true
 
-      # Check compliance
-      raise RestFtpDaemon::JobTargetUnparseable if @target_url.nil?
-      raise RestFtpDaemon::JobTargetUnsupported if @target_method.nil?
+      elsif target_uri.is_a? URI::SFTP
+        log_info "Job.prepare target_method SFTP"
+        set :target_method, :sftp
+        @remote = RemoteSFTP.new target_uri, log_context
+
+      else
+        log_info "Job.prepare unknown scheme [#{target_uri.scheme}]"
+        raise RestFtpDaemon::JobTargetUnsupported
+
+      end
     end
 
     def transfer
       # Update job status
-      newstatus :checking_source
+      #log_info "Job.transfer starting"
       @started_at = Time.now
 
       # Method assertions and init
       raise RestFtpDaemon::JobAssertionFailed, "transfer/1" unless @source_path
-      raise RestFtpDaemon::JobAssertionFailed, "transfer/2" unless @target_url
+      raise RestFtpDaemon::JobAssertionFailed, "transfer/2" unless @target_path
       @transfer_sent = 0
       set :source_processed, 0
 
-      # Guess source file names using Dir.glob
-      source_matches = Dir.glob @source_path
-
-      # Log detected names
-      source_names = source_matches.map{ |s| Helpers.extract_filename s }
-      log_info "Job.transfer sources #{source_names.inspect}"
-
-      # Asserts and counters
-      raise RestFtpDaemon::JobSourceNotFound if source_matches.empty?
-      set :source_count, source_matches.count
-      set :source_files, source_matches
+      # Guess source files from disk
+      newstatus :checking_source
+      sources = find_local @source_path
+      set :source_count, sources.count
+      set :source_files, sources.collect(&:full)
+      log_info "Job.transfer sources #{sources.collect(&:name)}"
+      #log_info "Job.transfer target #{target.full}"
+      raise RestFtpDaemon::JobSourceNotFound if sources.empty?
 
       # Guess target file name, and fail if present while we matched multiple sources
-      target_name = Helpers.extract_filename @target_url.path
-      raise RestFtpDaemon::JobTargetShouldBeDirectory if target_name && source_matches.count>1
-
-      # Scheme-aware config
-      ftp_init
+      raise RestFtpDaemon::JobTargetDirectoryError if @target_path.name && sources.count>1
 
       # Connect to remote server and login
-      ftp_connect
-      ftp_login
-
-      # Change to the right path
-      path = Helpers.extract_dirname(@target_url.path).to_s
-      ftp_chdir_or_buildpath path
+      newstatus :remote_connect
+      #log_info "Job.remote_connect" # [#{host}] [#{login}]"
+      @remote.connect
 
-      # Check source files presence and compute total size, they should be there, coming from Dir.glob()
-      @transfer_total = 0
-      source_matches.each do |filename|
-        raise RestFtpDaemon::JobSourceNotReadable unless File.readable? filename
-        raise RestFtpDaemon::JobSourceNotReadable unless File.file? filename
+      # Prepare target path or build it if asked
+      #log_info "Job.remote_chdir"
+      newstatus :remote_chdir
+      @remote.chdir_or_create @target_path.dir, @mkdir
 
-        # Skip if not a flie
-        next unless File.file? filename
-
-        @transfer_total += File.size filename
-      end
+      # Compute total files size
+      @transfer_total = sources.collect(&:size).sum
       set :transfer_total, @transfer_total
 
+      # Reset counters
+      @last_data = 0
+      @last_time = Time.now
+
       # Handle each source file matched, and start a transfer
-      done = 0
-      source_matches.each do |filename|
-        # Do the transfer, only if it's a file
-        ftp_transfer filename, target_name
+      source_processed = 0
+      sources.each do |source|
+        # Compute target filename
+        full_target = @target_path.clone
+
+        # Add the source file name if none found in the target path
+        unless full_target.name
+          full_target.name = source.name
+        end
+
+        # Do the transfer, for each file
+        #log_info "Job.remote_push"
+        remote_push source, full_target
 
         # Update counters
-        done += 1
-        set :source_processed, done
+        set :source_processed, source_processed += 1
       end
 
       # FTP transfer finished
-      ftp_finished
+      finalize
     end
 
 
@@ -383,6 +379,18 @@ module RestFtpDaemon
       }
     end
 
+    def find_local path
+      Dir.glob(path).collect do |file|
+        next unless File.readable? file
+        next unless File.file? file
+        Path.new file
+      end
+    end
+
+    def worker_is_still_active
+       Thread.current.thread_variable_set :updted_at, Time.now
+    end
+
     def newstatus name
       @status = name
       worker_is_still_active
@@ -399,57 +407,13 @@ module RestFtpDaemon
       instance_variable_set variable, default
     end
 
-    def ftp_init
-      # Update job status
-      newstatus :ftp_init
-
-      # Method assertions
-      raise RestFtpDaemon::JobAssertionFailed, "ftp_init/1" if @target_method.nil?
-      raise RestFtpDaemon::JobAssertionFailed, "ftp_init/2" if @target_url.nil?
-
-      log_info "Job.ftp_init target_method [#{@target_method}]"
-      case @target_method
-      when :ftp
-        @ftp = Net::FTP.new
-      when :ftps
-        @ftp = DoubleBagFTPS.new
-        @ftp.ssl_context = DoubleBagFTPS.create_ssl_context(verify_mode: OpenSSL::SSL::VERIFY_NONE)
-        @ftp.ftps_mode = DoubleBagFTPS::EXPLICIT
-      else
-        log_info "Job.ftp_init unknown scheme [#{@target_url.scheme}]"
-        railse RestFtpDaemon::JobTargetUnsupported
-      end
-
-      # FTP debug mode ?
-      if @ftp_debug_enabled
-        # Output header to STDOUT
-        puts
-        puts "-------------------- FTP SESSION STARTING --------------------"
-        puts "job id\t #{@id}"
-        puts "source\t #{@source}"
-        puts "target\t #{@target}"
-        puts "host\t #{@target_url.host}"
-        puts "user\t #{@target_url.user}"
-        puts "--------------------------------------------------------------"
-
-        # Set debug mode on connection
-        @ftp.debug_mode = true
-      end
-
-      # Activate passive mode
-      @ftp.passive = true
-    end
-
-    def ftp_finished
+    def finalize
       # Close FTP connexion and free up memory
-      @ftp.close
-      log_info "Job.ftp_finished closed"
-      @ftp = nil
+      log_info "Job.finalize"
+      @remote.close
 
-      # FTP debug mode ?
-      if @ftp_debug_enabled
-        puts "-------------------- FTP SESSION ENDED -----------------------"
-      end
+      # Free-up remote object
+      @remote = nil
 
       # Update job status
       newstatus :disconnecting
@@ -460,206 +424,106 @@ module RestFtpDaemon
       $queue.counter_add :transferred, @transfer_total
     end
 
-    def ftp_connect
-      # Update job status
-      newstatus :ftp_connect
-
-      # Method assertions
-      host = @target_url.host
-      log_info "Job.ftp_connect [#{host}]"
-      raise RestFtpDaemon::JobAssertionFailed, "ftp_connect/1" if @ftp.nil?
-      raise RestFtpDaemon::JobAssertionFailed, "ftp_connect/2" if @target_url.nil?
-
-      @ftp.connect(host)
-    end
-
-    def ftp_login
-      # Update job status
-      newstatus :ftp_login
-
-      # Method assertions
-      raise RestFtpDaemon::JobAssertionFailed, "ftp_login/1" if @ftp.nil?
-
-      # use "anonymous" if user is empty
-      login = @target_url.user || "anonymous"
-      log_info "Job.ftp_login [#{login}]"
-
-      @ftp.login login, @target_url.password
-    end
-
-    def ftp_chdir_or_buildpath path
-      # Method assertions
-      log_info "Job.ftp_chdir [#{path}] mkdir: #{@mkdir}"
-      newstatus :ftp_chdir
-      raise RestFtpDaemon::JobAssertionFailed, "ftp_chdir_or_buildpath/1" if path.nil?
-
-      # Extract directory from path
-      if @mkdir
-        # Split dir in parts
-        log_info "Job.ftp_chdir buildpath [#{path}]"
-        ftp_buildpath path
-      else
-        # Directly chdir if not mkdir requested
-        log_info "Job.ftp_chdir chdir [#{path}]"
-        @ftp.chdir path
-      end
-    end
-
-    def ftp_buildpath path
-      # Init
-      pref = "Job.ftp_buildpath [#{path}]"
-
-      begin
-        # Try to chdir in this directory
-        @ftp.chdir(path)
-
-      rescue Net::FTPPermError
-        # If not possible because the directory is missing
-        parent =  Helpers.extract_parent(path)
-        log_info "#{pref} chdir failed - parent [#{parent}]"
-
-        # And only if we still have something to "dive up into"
-        if parent
-          # Do the same for the parent
-          ftp_buildpath parent
-
-          # Then finally create this dir and chdir
-          log_info "#{pref} > now mkdir [#{path}]"
-          @ftp.mkdir path
-
-          # And get into it (this chdir is not rescue'd on purpose in order to throw the ex)
-          log_info "#{pref} > now chdir [#{path}]"
-          @ftp.chdir(path)
-        end
-
-      end
-
-      # Now we were able to chdir inside, just tell it
-      log_info "#{pref} > ftp.pwd [#{@ftp.pwd}]"
-    end
-
-    def ftp_presence target_name
-      # Update job status
-      newstatus :ftp_presence
-# FIXME / TODO: try with nlst
-
-      # Method assertions
-      raise RestFtpDaemon::JobAssertionFailed, "ftp_presence/1" if @ftp.nil?
-      raise RestFtpDaemon::JobAssertionFailed, "ftp_presence/2" if @target_url.nil?
-
-      # Get file list, sometimes the response can be an empty value
-      results = @ftp.list(target_name) rescue nil
-      log_info "Job.ftp_presence: #{results.inspect}"
-
-      # Result can be nil or a list of files
-      return false if results.nil?
-
-      results.count >0
-    end
-
-    def ftp_transfer source_filename, target_name = nil
+    def remote_push source, target
       # Method assertions
-      log_info "Job.ftp_transfer source: #{source_filename}"
-      raise RestFtpDaemon::JobAssertionFailed, "ftp_transfer/1" if @ftp.nil?
-      raise RestFtpDaemon::JobAssertionFailed, "ftp_transfer/2" if source_filename.nil?
+      raise RestFtpDaemon::JobAssertionFailed, "ftp_transfer/1" if @remote.nil?
+      raise RestFtpDaemon::JobAssertionFailed, "ftp_transfer/2" if source.nil?
+      raise RestFtpDaemon::JobAssertionFailed, "ftp_transfer/3" if target.nil?
 
       # Use source filename if target path provided none (typically with multiple sources)
-      target_name ||= Helpers.extract_filename source_filename
-      log_info "Job.ftp_transfer target: #{target_name}"
-      set :source_current, target_name
-
-      # Check for target file presence
-      newstatus :checking_target
-      present = ftp_presence target_name
-      if present
-        if @overwrite
-          # delete it first
-          log_info "Job.ftp_transfer removing target file"
-          @ftp.delete(target_name)
-        else
-          # won't overwrite then stop here
-          log_info "Job.ftp_transfer failed: target file exists"
-          raise RestFtpDaemon::JobTargetFileExists
-        end
-      end
+      log_info "Job.remote_push [#{source.name}]: [#{source.full}] > [#{target.full}]"
+      set :source_current, source.name
 
       # Compute temp target name
-      target_real = target_name
+      tempname = nil
       if @tempfile
-        target_real = "#{target_name}.temp-#{Helpers.identifier(JOB_TEMPFILE_LEN)}"
-        log_info "Job.ftp_transfer tempfile: #{target_real}"
+        tempname = "#{target.name}.temp-#{Helpers.identifier(JOB_TEMPFILE_LEN)}"
+        #log_info "Job.remote_push tempname [#{tempname}]"
+      else
+      end
+
+      # Remove any existing version if expected, or test its presence
+      if @overwrite
+        @remote.remove! target
+      elsif size = @remote.present?(target)
+        log_info "Job.remote_push existing (#{Helpers.format_bytes size, 'B'})"
+        raise RestFtpDaemon::JobTargetFileExists
       end
 
       # Start transfer
       transfer_started_at = Time.now
-      @transfer_pointer_at = transfer_started_at
-
-      @notified_at = Time.now
+      @progress_at = 0
+      @notified_at = transfer_started_at
       newstatus JOB_STATUS_UPLOADING
 
-      @ftp.putbinaryfile(source_filename, target_real, @chunk_size) do |block|
-        # Update the worker activity marker
-        worker_is_still_active
-
-        # Update job status after this block transfer
-        ftp_transfer_block block
-      end
+      # Start the transfer, update job status after each block transfer
+      newstatus :uploading
+      @remote.push source, target, tempname do |transferred, name|
+        # Update transfer statistics
+        progress transferred, name
 
-      # Rename temp file to target_temp
-      if @tempfile
-        newstatus JOB_STATUS_RENAMING
-        log_info "Job.ftp_transfer renaming: #{target_name}"
-        @ftp.rename target_real, target_name
+        # Touch my worker status
+        worker_is_still_active
       end
 
       # Compute final bitrate
-      set :transfer_bitrate, get_bitrate(@transfer_total, transfer_started_at).round(0)
+      global_transfer_bitrate = get_bitrate @transfer_total, (Time.now - transfer_started_at)
+      set :transfer_bitrate, global_transfer_bitrate.round(0)
 
       # Done
       set :source_current, nil
-      log_info "Job.ftp_transfer finished"
+      #log_info "Job.remote_push finished"
     end
 
-    def ftp_transfer_block block
+    def progress transferred, name = ""
+      # What's current time ?
+      now = Time.now
+
       # Update counters
-      @transfer_sent += block.bytesize
+      @transfer_sent += transferred
       set :transfer_sent, @transfer_sent
 
-      # Update bitrate
-      #dt = Time.now - t0
-      bitrate0 = get_bitrate(@chunk_size, @transfer_pointer_at).round(0)
-      set :transfer_bitrate, bitrate0
-
       # Update job info
       percent0 = (100.0 * @transfer_sent / @transfer_total).round(0)
       set :progress, percent0
 
-      # Log progress
-      stack = []
-      stack << "#{percent0} %"
-      stack << (Helpers.format_bytes @transfer_sent, "B")
-      stack << (Helpers.format_bytes @transfer_total, "B")
-      stack << (Helpers.format_bytes bitrate0, "bps")
-      stack2 = stack.map{ |txt| ("%#{LOG_PIPE_LEN.to_i}s" % txt)}.join("\t")
-      log_info "Job.ftp_transfer #{stack2}"
-
-      # Update time pointer
-      @transfer_pointer_at = Time.now
+      # Update job status after each NOTIFY_UPADE_STATUS
+      progressed_ago = (now.to_f - @progress_at.to_f)
+      if (!JOB_UPDATE_INTERVAL.to_f.zero?) && (progressed_ago > JOB_UPDATE_INTERVAL.to_f)
+        @current_bitrate = running_bitrate @transfer_sent
+        set :transfer_bitrate, @current_bitrate.round(0)
+
+        # Log progress
+        stack = []
+        stack << "#{percent0} %"
+        stack << (Helpers.format_bytes @transfer_sent, "B")
+        stack << (Helpers.format_bytes @transfer_total, "B")
+        stack << (Helpers.format_bytes @current_bitrate.round(0), "bps")
+        stack2 = stack.map{ |txt| ("%#{LOG_PIPE_LEN.to_i}s" % txt)}.join("\t")
+        log_info "#{LOG_INDENT}progress #{stack2} \t#{name}"
+
+        # Remember when we last did it
+        @progress_at = now
+      end
 
       # Notify if requested
-      # info "Job.ftp_transfer next notif (#{(@notified_at+@notify_after_sec).to_f}) now #{Time.now.to_f}"
-      if @notify_after_sec.nil? || (Time.now > @notified_at + @notify_after_sec)
+      notified_ago = (now.to_f - @notified_at.to_f)
+      if (!@notify_after_sec.nil?) && (notified_ago > @notify_after_sec)
+        # Prepare and send notification
         notif_status = {
           progress: percent0,
           transfer_sent: @transfer_sent,
           transfer_total: @transfer_total,
-          transfer_bitrate: bitrate0
+          transfer_bitrate: @current_bitrate
           }
         client_notify :progress, status: notif_status
-        @notified_at = Time.now
+
+        # Remember when we last did it
+        @notified_at = now
       end
     end
 
+
     def client_notify event, payload = {}
       # Skip if no URL given
       return unless @notify
@@ -673,12 +537,25 @@ module RestFtpDaemon
       log_error "Job.client_notify EXCEPTION: #{ex.inspect}"
      end
 
-    def get_bitrate total, last_timestamp
-      8*total.to_f / (Time.now - last_timestamp)
+    def get_bitrate delta_data, delta_time
+      return nil if delta_time.nil? || delta_time.zero?
+      8 * delta_data.to_f.to_f / delta_time
     end
 
-    def worker_is_still_active
-       Thread.current.thread_variable_set :updted_at, Time.now
+    def running_bitrate current_data
+      return if @last_time.nil?
+
+      # Compute deltas
+      @last_data ||= 0
+      delta_data = current_data - @last_data
+      delta_time = Time.now - @last_time
+
+      # Update counters
+      @last_time = Time.now
+      @last_data = current_data
+
+      # Return bitrate
+      get_bitrate delta_data, delta_time
     end
 
     def oops event, exception, error = nil, include_backtrace = false
@@ -693,7 +570,7 @@ module RestFtpDaemon
       end
 
       # Close ftp connexion if open
-      @ftp.close unless @ftp.nil? || @ftp.welcome.nil?
+      @remote.close unless @remote.nil? || !@remote.connected?
 
       # Update job's internal status
       newstatus JOB_STATUS_FAILED