rest-ftp-daemon 0.90.1 → 0.94.4

data/lib/rest-ftp-daemon/api/root.rb

@@ -1,140 +1,72 @@
-require 'haml'
-require "facter"
-require "sys/cpu"
-
-
 module RestFtpDaemon
   module API
-
     class Root < Grape::API
 
 
 ####### CLASS CONFIG
 
-      include RestFtpDaemon::API::Defaults
       logger RestFtpDaemon::Logger.new(:api, "API")
-      mount RestFtpDaemon::API::Jobs => '/jobs'
-      #add_swagger_documentation
-      # mount RestFtpDaemon::API::Workers => '/workers'
 
+      do_not_route_head!
+      do_not_route_options!
 
-####### HELPERS
-
-      helpers do
-        def info message, level = 0
-          Root.logger.info(message, level)
-        end
+      # FIXME
+      # add_swagger_documentation
+      # default_error_formatter :json
+      format :json
 
-      end
-
-
-####### INITIALIZATION
-
-      def initialize
-        # Check that Queue and Pool are available
-        raise RestFtpDaemon::MissingQueue unless defined? $queue
-        raise RestFtpDaemon::MissingQueue unless defined? $pool
-        super
-      end
 
+####### EXCETPIONS HANDLING
+      # FIXME
+      # rescue_from :all do |e|
+      #   error_response(message: "Internal server error", status: 500)
+      # end
 
-####### API DEFINITION
 
-      # Server global status
-      get '/' do
-        info "GET /"
+####### HELPERS
 
-        # Initialize UsageWatch
-        Facter.loadfacts
-        @info_load = Sys::CPU.load_avg.first.to_f
-        @info_procs = (Facter.value :processorcount).to_i
-        @info_ipaddr = Facter.value(:ipaddress)
-        @info_memfree = Facter.value(:memoryfree)
+      helpers do
 
-        # Compute normalized load
-        if @info_procs.zero?
-          @info_norm = "N/A"
-        else
-          @info_norm = (100 * @info_load / @info_procs).round(1)
+        def info message, level = 0
+          Root.logger.info(message, level)
         end
 
-        # Jobs to display
-        # all_jobs_in_queue = $queue.all
-        # all_jobs_in_queue = $queue.ordered_queue
-        # + $queue.popped
-        popped_jobs = $queue.ordered_popped.reverse
-        @jobs_queued = $queue.ordered_queue.reverse
-
-        if params["only"].nil? || params["only"].blank?
-          @only = nil
-        else
-          @only = params["only"].to_sym
+        def api_error exception
+          {
+          :error => exception.message,
+          :message => exception.backtrace.first,
+          #:backtrace => exception.backtrace,
+          }
         end
 
-        case @only
-        when nil
-          @jobs_popped = popped_jobs
-        # when :queue
-        #   @jobs_popped = $queue.queued
-        else
-          @jobs_popped = $queue.popped_reverse_sorted_by_status @only
+        def render name, values={}
+          template = File.read("#{APP_LIBS}/views/#{name.to_s}.haml")
+          haml_engine = Haml::Engine.new(template)
+          haml_engine.render(binding, values)
         end
 
-        # Count jobs for each status
-        @counts = $queue.popped_counts_by_status
+        def job_find job_id
+          return nil if ($queue.all_size==0)
 
-        # Get workers status
-        @gworker_statuses = $pool.get_worker_statuses
+          # Find a job with exactly this id, or prefixed if not found
+          $queue.find_by_id(job_id) || $queue.find_by_id(job_id, true)
+        end
 
-        # Compile haml template
-        output = render :dashboard
+      end
 
-        # Send response
-        env['api.format'] = :html
-        format "html"
-        status 200
-        content_type "text/html"
-        body output
 
-      end
+####### INITIALIZATION
 
-      # Server global status
-      get '/status' do
-        info "GET /status"
-        status 200
-        return  {
-          hostname: `hostname`.chomp,
-          version: APP_VER,
-          config: Settings.to_hash,
-          started: APP_STARTED,
-          uptime: (Time.now - APP_STARTED).round(1),
-          counters: $queue.counters,
-          status: $queue.popped_counts_by_status,
-          queue_size: $queue.all_size,
-          jobs_queued: $queue.queued.collect(&:id),
-          jobs_popped: $queue.popped.collect(&:id),
-          routes: RestFtpDaemon::API::Root::routes,
-          }
-      end
+      def initialize
+        # Call daddy
+        super
 
-      # Server test
-      get '/debug' do
-        info "GET /debug"
-
-        begin
-          raise RestFtpDaemon::DummyException
-        rescue RestFtpDaemon::RestFtpDaemonException => exception
-          status 501
-          api_error exception
-        rescue Exception => exception
-          status 501
-          api_error exception
-        else
-          status 200
-          {}
-        end
+        # Check that Queue and Pool are available
+        raise RestFtpDaemon::MissingQueue unless defined? $queue
+        raise RestFtpDaemon::MissingQueue unless defined? $pool
       end
 
+
     end
   end
 end

data/lib/rest-ftp-daemon/api/routes.rb

@@ -0,0 +1,16 @@
+module RestFtpDaemon
+  module API
+    class Root < Grape::API
+
+
+####### GET /routes
+
+      get '/routes' do
+        info "GET /routes"
+        status 200
+        return RestFtpDaemon::API::Root::routes
+      end
+
+    end
+  end
+end

data/lib/rest-ftp-daemon/api/status.rb

@@ -0,0 +1,29 @@
+module RestFtpDaemon
+  module API
+    class Root < Grape::API
+
+
+####### GET /status
+
+      # Server global status
+      get '/status' do
+        info "GET /status"
+        status 200
+        return  {
+          hostname: `hostname`.chomp,
+          version: APP_VER,
+          config: Helpers.get_censored_config,
+          started: APP_STARTED,
+          uptime: (Time.now - APP_STARTED).round(1),
+          counters: $queue.counters,
+          status: $queue.counts_by_status,
+          jobs_count: $queue.all_size,
+          jobs_queued: $queue.queued.collect(&:id),
+          jobs_popped: $queue.popped.collect(&:id),
+          #routes: RestFtpDaemon::API::Root::routes,
+          }
+      end
+
+    end
+  end
+end

data/lib/rest-ftp-daemon/constants.rb

@@ -1,8 +1,7 @@
 # Terrific constants
 APP_NAME = "rest-ftp-daemon"
 APP_CONF = "/etc/#{APP_NAME}.yml"
-APP_VER = "0.90.1"
-
+APP_VER = "0.94.4"
 
 # Some global constants
 IDENT_JOB_LEN = 4
@@ -15,7 +14,10 @@ DEFAULT_UPDATE_EVERY_KB = 2048
 DEFAULT_WORKERS = 1
 DEFAULT_LOGS_PROGNAME_TRIM = 12
 
-
 # Initialize markers
 APP_STARTED = Time.now
 APP_LIBS = File.dirname(__FILE__)
+
+# Debugging
+DEBUG_FTP_COMMANDS = false
+

data/lib/rest-ftp-daemon/helpers.rb

@@ -3,6 +3,13 @@ require 'securerandom'
 module RestFtpDaemon
   class Helpers
 
+    def self.get_censored_config
+      config = Settings.to_hash
+      config[:users] = Settings.users.keys if Settings.users
+      config[:endpoints] = Settings.endpoints.keys if Settings.endpoints
+      config
+    end
+
     def self.format_bytes number, unit=""
       return "Ø" if number.nil? || number.zero?
 
@@ -12,6 +19,11 @@ module RestFtpDaemon
       "#{converted} #{units[index]}#{unit}"
     end
 
+    def self.text_or_empty text
+      return "Ø" if text.nil? || text.empty?
+      text
+    end
+
     def self.identifier len
       rand(36**len).to_s(36)
     end
@@ -20,12 +32,25 @@ module RestFtpDaemon
       "[#{item}]"
     end
 
-    def self.contains_tokens(item)
-      /\[[a-zA-Z0-9]+\]/.match(item)
+    def self.highlight_tokens(path)
+      path.gsub(/(\[[^\[]+\])/, '<span class="token">\1</span>')
     end
 
     def self.extract_filename path
-      m = path.match /\/([^\/]+)$/
+      # match everything that's after a slash at the end of the string
+      m = path.match /\/?([^\/]+)$/
+      return m[1] unless m.nil?
+    end
+
+    def self.extract_dirname path
+      # match all the beginning of the string up to the last slash
+      m = path.match(/^(.*)\/[^\/]*$/)
+      return "/#{m[1]}" unless m.nil?
+    end
+
+    def self.extract_parent path
+      # m = path.match(/^(.*\/)[^\/]*\/+$/)
+      m = path.match(/^(.*\/)[^\/]+\/?$/)
       return m[1] unless m.nil?
     end
 
@@ -65,15 +90,16 @@ module RestFtpDaemon
       "<div class=\"transfer-method label #{klass}\">#{method.upcase}</div>"
     end
 
-    # def snakecase
-    #   gsub(/::/, '/').
-    #   gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
-    #   gsub(/([a-z\d])([A-Z])/,'\1_\2').
-    #   tr('-', '_').
-    #   gsub(/\s/, '_').
-    #   gsub(/__+/, '_').
-    #   downcase
-    # end
+    # Dates and times: date with time generator
+    def self.datetime_full param
+      return "-"  if param.nil?
+      return param.to_datetime.strftime("%d.%m.%Y %H:%M")
+    end
+
+    def self.datetime_short param
+      return "-"  if param.nil?
+      return param.to_datetime.strftime("%d/%m %H:%M")
+    end
 
   end
 end

data/lib/rest-ftp-daemon/job.rb

@@ -1,5 +1,3 @@
-#require 'net/ftptls'
-
 require 'uri'
 require 'net/ftp'
 require 'double_bag_ftps'
@@ -7,18 +5,36 @@ require 'timeout'
 
 module RestFtpDaemon
   class Job < RestFtpDaemon::Common
-    attr_accessor :id
+
+    FIELDS = [:source, :target, :priority, :notify, :overwrite, :mkdir]
+
+    attr_reader :id
     attr_accessor :wid
 
+    attr_reader :error
+    attr_reader :status
+
+    attr_reader :started_at
+    attr_reader :updated_at
+
+    attr_reader :params
+
+    FIELDS.each do |field|
+      attr_reader field
+    end
+
     def initialize job_id, params={}
       # Call super
       # super()
       info "Job.initialize"
 
       # Init context
-      @params = params
       @id = job_id.to_s
       #set :id, job_id
+      FIELDS.each do |field|
+        instance_variable_set("@#{field.to_s}", params[field])
+      end
+      @params = {}
 
       # Protect with a mutex
       @mutex = Mutex.new
@@ -27,40 +43,29 @@ module RestFtpDaemon
       @logger = RestFtpDaemon::Logger.new(:workers, "JOB #{id}")
 
       # Flag current job
-      set :started_at, Time.now
-      status :created
+      @started_at = Time.now
+      @status = :created
 
       # Send first notification
       #info "Job.initialize/notify"
-      notify "rftpd.queued"
-    end
-
-    def id
-      @id
-    end
-
-    def priority
-      get :priority
-    end
-    def get_status
-      get :status
+      client_notify "rftpd.queued"
     end
 
     def process
       # Update job's status
-      set :error, nil
+      @error = nil
 
       # Prepare job
       begin
-        info "Job.process/prepare"
-        status :preparing
+        info "Job.process prepare"
+        @status = :preparing
         prepare
 
       rescue RestFtpDaemon::JobMissingAttribute => exception
         return oops "rftpd.started", exception, :job_missing_attribute
 
-      rescue RestFtpDaemon::JobSourceNotFound => exception
-        return oops "rftpd.started", exception, :job_source_not_found
+      # rescue RestFtpDaemon::JobSourceNotFound => exception
+      #   return oops "rftpd.started", exception, :job_source_not_found
 
       rescue RestFtpDaemon::JobUnresolvedTokens => exception
         return oops "rftpd.started", exception, :job_unresolved_tokens
@@ -68,33 +73,32 @@ module RestFtpDaemon
       rescue RestFtpDaemon::JobTargetUnparseable => exception
         return oops "rftpd.started", exception, :job_target_unparseable
 
+      rescue RestFtpDaemon::JobTargetUnsupported => exception
+        return oops "rftpd.started", exception, :job_target_unsupported
+
       rescue RestFtpDaemon::JobAssertionFailed => exception
         return oops "rftpd.started", exception, :job_assertion_failed
 
       rescue RestFtpDaemon::RestFtpDaemonException => exception
         return oops "rftpd.started", exception, :job_prepare_failed
 
+      rescue URI::InvalidURIError => exception
+        return oops "rftpd.started", exception, :job_target_invalid
+
       rescue Exception => exception
         return oops "rftpd.started", exception, :job_prepare_unhandled, true
 
       else
-        # Update job's status
-        info "Job.process/prepare ok"
-        status :prepared
-        info "Job.process/prepare status updated"
-
-        # Notify rftpd.start
-        info "Job.process/prepare notify started"
-        notify "rftpd.started", 0
-        info "Job.process/prepare notified started"
+        # Prepare done !
+        @status = :prepared
+        info "Job.process notify rftpd.started"
+        client_notify "rftpd.started", nil
       end
 
-      info "Job.process prepare>transfer"
-
       # Process job
       begin
-        info "Job.process/transfer"
-        status :starting
+        info "Job.process transfer"
+        @status = :starting
         transfer
 
       rescue Errno::EHOSTDOWN => exception
@@ -109,9 +113,15 @@ module RestFtpDaemon
       rescue Net::FTPPermError => exception
         return oops "rftpd.ended", exception, :job_perm_error
 
+      rescue OpenSSL::SSL::SSLError => exception
+        return oops "rftpd.ended", exception, :job_openssl_error
+
       rescue Errno::EMFILE => exception
         return oops "rftpd.ended", exception, :job_too_many_open_files
 
+      rescue RestFtpDaemon::JobSourceNotFound => exception
+        return oops "rftpd.ended", exception, :job_source_not_found
+
       rescue RestFtpDaemon::JobTargetFileExists => exception
         return oops "rftpd.ended", exception, :job_target_file_exists
 
@@ -128,31 +138,14 @@ module RestFtpDaemon
         return oops "rftpd.ended", exception, :job_transfer_unhandled, true
 
       else
-        # Update job's status
-        info "Job.process finished"
-        status :finished
-
-        # Notify rftpd.ended
-        notify "rftpd.ended", 0
+        # All done !
+        @status = :finished
+        info "Job.process notify rftpd.ended"
+        client_notify "rftpd.ended", nil
       end
 
     end
 
-    def describe
-      # Update realtime info
-      #u = up_time
-      #set :uptime, u.round(2) unless u.nil?
-
-      # Return the whole structure  FIXME
-      @params.merge({
-        id: @id,
-        uptime: up_time.round(2)
-        })
-      # @mutex.synchronize do
-      #   out = @params.clone
-      # end
-    end
-
     def get attribute
       @mutex.synchronize do
         @params || {}
@@ -160,11 +153,15 @@ module RestFtpDaemon
       end
     end
 
+    def set_queued
+      @status = :queued
+    end
+
   protected
 
-    def up_time
-      return 0 if @params[:started_at].nil?
-      Time.now - @params[:started_at]
+    def age
+      return 0 if @started_at.nil?
+      (Time.now - @started_at).round(2)
     end
 
     def wander time
@@ -184,21 +181,21 @@ module RestFtpDaemon
       @mutex.synchronize do
         @params || {}
         # return unless @params.is_a? Enumerable
-        @params[:updated_at] = Time.now
+        @updated_at = Time.now
         @params[attribute] = value
       end
     end
 
-    def status status
-      set :status, status
-    end
-
     def expand_path path
       File.expand_path replace_tokens(path)
     end
 
     def expand_url path
-      URI::parse replace_tokens(path) rescue nil
+      URI::parse replace_tokens(path)
+    end
+
+    def contains_brackets(item)
+      /\[.*\]/.match(item)
     end
 
     def replace_tokens path
@@ -213,34 +210,33 @@ module RestFtpDaemon
       newpath = path.clone
       vectors.each do |from, to|
         next if to.to_s.blank?
-        #info "Job.replace_tokens #{Helpers.tokenize(from)} > #{to}"
         newpath.gsub! Helpers.tokenize(from), to
       end
 
       # Ensure result does not contain tokens after replacement
-      raise RestFtpDaemon::JobUnresolvedTokens if Helpers.contains_tokens newpath
+      raise RestFtpDaemon::JobUnresolvedTokens if contains_brackets newpath
 
-      # All OK, return this
-      return newpath
+      # All OK, return this URL stripping multiple slashes
+      return newpath.gsub(/([^:])\/\//, '\1/')
     end
 
     def prepare
       # Init
-      status :preparing
+      @status = :preparing
       @source_method = :file
       @target_method = nil
       @source_path = nil
       @target_url = nil
 
       # Check source
-      raise RestFtpDaemon::JobMissingAttribute unless @params[:source]
-      @source_path = expand_path @params[:source]
+      raise RestFtpDaemon::JobMissingAttribute unless @source
+      @source_path = expand_path @source
       set :source_path, @source_path
       set :source_method, :file
 
       # Check target
-      raise RestFtpDaemon::JobMissingAttribute unless @params[:target]
-      @target_url = expand_url @params[:target]
+      raise RestFtpDaemon::JobMissingAttribute unless @target
+      @target_url = expand_url @target
       set :target_url, @target_url.to_s
 
       if @target_url.kind_of? URI::FTP
@@ -255,22 +251,21 @@ module RestFtpDaemon
       # Check compliance
       raise RestFtpDaemon::JobTargetUnparseable if @target_url.nil?
       raise RestFtpDaemon::JobTargetUnsupported if @target_method.nil?
-      #raise RestFtpDaemon::JobSourceNotFound unless File.exists? @source_path
     end
 
     def transfer
       # Method assertions and init
-      status :checking_source
+      @status = :checking_source
       raise RestFtpDaemon::JobAssertionFailed unless @source_path && @target_url
       @transfer_sent = 0
-      set :transfer_source_done, 0
+      set :source_processed, 0
 
       # Guess source file names using Dir.glob
       source_matches = Dir.glob @source_path
-      info "Job.transfer sources: #{source_matches}"
+      info "Job.transfer sources #{source_matches.inspect}"
       raise RestFtpDaemon::JobSourceNotFound if source_matches.empty?
-      set :transfer_source_count, source_matches.count
-      set :transfer_source_files, source_matches
+      set :source_count, source_matches.count
+      set :source_files, source_matches
 
       # Guess target file name, and fail if present while we matched multiple sources
       target_name = Helpers.extract_filename @target_url.path
@@ -280,7 +275,12 @@ module RestFtpDaemon
       ftp_init
 
       # Connect remote server, login and chdir
-      ftp_connect
+      ftp_connect_and_login
+
+      # Connect remote server, login and chdir
+      #path = '/' + Helpers.extract_dirname(@target_url.path).to_s
+      path = Helpers.extract_dirname(@target_url.path).to_s
+      ftp_chdir_or_buildpath path
 
       # Check source files presence and compute total size, they should be there, coming from Dir.glob()
       @transfer_total = 0
@@ -296,7 +296,7 @@ module RestFtpDaemon
       source_matches.each do |filename|
         ftp_transfer filename, target_name
         done += 1
-        set :transfer_source_done, done
+        set :source_processed, done
       end
 
       # Add total transferred to counter
@@ -304,7 +304,7 @@ module RestFtpDaemon
 
       # Close FTP connexion
       info "Job.transfer disconnecting"
-      status :disconnecting
+      @status = :disconnecting
       @ftp.close
     end
 
@@ -316,68 +316,127 @@ module RestFtpDaemon
       info "Job.oops si[#{signal_name}] er[#{error_name.to_s}] ex[#{exception.class}]"
 
       # Update job's internal status
-      set :status, :failed
-      set :error, error_name
+      @status = :failed
+      @error = error_name
+      set :error_name, error_name
       set :error_exception, exception.class
+      set :error_message, exception.message
 
       # Build status stack
-      status = nil
+      notif_status = nil
       if include_backtrace
         set :error_backtrace, exception.backtrace
-        status = {
+        notif_status = {
           backtrace: exception.backtrace,
         }
       end
 
+      # Increment counter for this error
+      $queue.counter_inc "err_#{error_name}"
+      $queue.counter_inc :jobs_failed
+
       # Prepare notification if signal given
       return unless signal_name
-      notify signal_name, error_name, status
+      client_notify signal_name, error_name, notif_status
     end
 
     def ftp_init
       # Method assertions
-      info "Job.ftp_init"
-      status :ftp_init
+      info "Job.ftp_init asserts"
+      @status = :ftp_init
       raise RestFtpDaemon::JobAssertionFailed if @target_method.nil? || @target_url.nil?
 
+      info "Job.ftp_init target_method [#{@target_method}]"
       case @target_method
       when :ftp
-        info "Job.ftp_init scheme: ftp"
         @ftp = Net::FTP.new
       when :ftps
-        info "Job.transfer scheme: ftps"
         @ftp = DoubleBagFTPS.new
         @ftp.ssl_context = DoubleBagFTPS.create_ssl_context(:verify_mode => OpenSSL::SSL::VERIFY_NONE)
         @ftp.ftps_mode = DoubleBagFTPS::EXPLICIT
       else
-        info "Job.transfer scheme: other [#{@target_url.scheme}]"
+        info "Job.transfer unknown scheme [#{@target_url.scheme}]"
+        railse RestFtpDaemon::JobTargetUnsupported
       end
+
+      # Common setup
+      @ftp.debug_mode = DEBUG_FTP_COMMANDS
+      @ftp.passive = true
     end
 
-    def ftp_connect
-      #status :ftp_connect
+    def ftp_connect_and_login
+      #@status = :ftp_connect
       # connect_timeout_sec = (Settings.transfer.connect_timeout_sec rescue nil) || DEFAULT_CONNECT_TIMEOUT_SEC
 
       # Method assertions
-      info "Job.ftp_connect connect"
-      status :ftp_connect
+      host = @target_url.host
+      info "Job.ftp_connect connect [#{host}]"
+      @status = :ftp_connect
       raise RestFtpDaemon::JobAssertionFailed if @ftp.nil? || @target_url.nil?
-      @ftp.connect(@target_url.host)
-      @ftp.passive = true
+      @ftp.connect(host)
 
-      info "Job.ftp_connect login"
-      status :ftp_login
-      ret = @ftp.login @target_url.user, @target_url.password
+      @status = :ftp_login
+      info "Job.ftp_connect login [#{@target_url.user}]"
+      @ftp.login @target_url.user, @target_url.password
+    end
 
-      info "Job.ftp_connect chdir"
-      status :ftp_chdir
-      path = File.dirname @target_url.path
-      ret = @ftp.chdir(path)
+    def ftp_chdir_or_buildpath path
+      # Method assertions
+      info "Job.ftp_chdir [#{path}] mkdir: #{@mkdir}"
+      raise RestFtpDaemon::JobAssertionFailed if path.nil?
+      @status = :ftp_chdir
+
+      # Extract directory from path
+      if @mkdir
+        # Split dir in parts
+        info "Job.ftp_chdir buildpath [#{path}]"
+        ftp_buildpath path
+      else
+        # Directly chdir if not mkdir requested
+        info "Job.ftp_chdir chdir [#{path}]"
+        @ftp.chdir path
+      end
     end
 
+    def ftp_buildpath path
+      # Init
+      pref = "Job.ftp_buildpath [#{path}]"
+
+      begin
+        # Try to chdir in this directory
+        @ftp.chdir(path)
+
+      rescue Net::FTPPermError => exception
+        # If not possible because the directory is missing
+        parent =  Helpers.extract_parent(path)
+        info "#{pref} chdir failed - parent [#{parent}]"
+
+        # And only if we still have something to "dive up into"
+        if parent
+          # Do the same for the parent
+          ftp_buildpath parent
+
+          # Then finally create this dir and chdir
+          info "#{pref} > now mkdir [#{path}]"
+          @ftp.mkdir path
+
+          # And get into it (this chdir is not rescue'd on purpose in order to throw the ex)
+          info "#{pref} > now chdir [#{path}]"
+          @ftp.chdir(path)
+        end
+
+      end
+
+      # Now we were able to chdir inside, just tell it
+      info "#{pref} changed to [#{@ftp.pwd}]"
+    end
+
+
     def ftp_presence target_name
+# FIXME / TODO: try with nlst
+
       # Method assertions
-      status :ftp_presence
+      @status = :ftp_presence
       raise RestFtpDaemon::JobAssertionFailed if @ftp.nil? || @target_url.nil?
 
       # Get file list, sometimes the response can be an empty value
@@ -390,24 +449,21 @@ module RestFtpDaemon
     end
 
     def ftp_transfer source_match, target_name = nil
-#target_name
       # Method assertions
-      info "Job.ftp_transfer source_match: #{source_match}"
+      info "Job.ftp_transfer source_match [#{source_match}]"
       raise RestFtpDaemon::JobAssertionFailed if @ftp.nil?
       raise RestFtpDaemon::JobAssertionFailed if source_match.nil?
-      #raise RestFtpDaemon::JobAssertionFailed if @transfer_total.nil?
-      status :ftp_transfer
 
       # Use source filename if target path provided none (typically with multiple sources)
       target_name ||= Helpers.extract_filename source_match
-      info "Job.ftp_transfer target_name: #{target_name}"
+      info "Job.ftp_transfer target_name [#{target_name}]"
+      set :source_processing, target_name
 
       # Check for target file presence
-      status :checking_target
-      overwrite = !get(:overwrite).nil?
+      @status = :checking_target
       present = ftp_presence target_name
       if present
-        if overwrite
+        if @overwrite
           # delete it first
           info "Job.ftp_transfer removing target file"
           @ftp.delete(target_name)
@@ -427,7 +483,7 @@ module RestFtpDaemon
       chunk_size = update_every_kb * 1024
       t0 = tstart = Time.now
       notified_at = Time.now
-      status :uploading
+      @status = :uploading
       @ftp.putbinaryfile(source_match, target_name, chunk_size) do |block|
         # Update counters
         @transfer_sent += block.bytesize
@@ -443,42 +499,41 @@ module RestFtpDaemon
         set :progress, percent1
 
         # Log progress
-        status = []
-        status << "#{percent1} %"
-        status << (Helpers.format_bytes @transfer_sent, "B")
-        status << (Helpers.format_bytes @transfer_total, "B")
-        status << (Helpers.format_bytes bitrate0, "bps")
-        info "Job.ftp_transfer" + status.map{|txt| ("%#{DEFAULT_LOGS_PROGNAME_TRIM.to_i}s" % txt)}.join("\t")
+        stack = []
+        stack << "#{percent1} %"
+        stack << (Helpers.format_bytes @transfer_sent, "B")
+        stack << (Helpers.format_bytes @transfer_total, "B")
+        stack << (Helpers.format_bytes bitrate0, "bps")
+        info "Job.ftp_transfer" + stack.map{|txt| ("%#{DEFAULT_LOGS_PROGNAME_TRIM.to_i}s" % txt)}.join("\t")
 
         # Update time pointer
         t0 = Time.now
 
         # Notify if requested
         unless notify_after_sec.nil? || (notified_at + notify_after_sec > Time.now)
-          status = {
+          notif_status = {
             progress: percent1,
             transfer_sent: @transfer_sent,
             transfer_total: @transfer_total,
             transfer_bitrate: bitrate0
             }
-          notify "rftpd.progress", 0, status
+          client_notify "rftpd.progress", nil, notif_status
           notified_at = Time.now
         end
 
       end
 
       # Compute final bitrate
-      #tbitrate0 = (8 * @transfer_total.to_f / (Time.now - tstart)).round(0)
-      tbitrate0 = get_bitrate(@transfer_total, tstart).round(0)
-      set :transfer_bitrate, tbitrate0
+      set :transfer_bitrate, get_bitrate(@transfer_total, tstart).round(0)
 
       # Done
-      #set :progress, nil
+      set :source_processing, nil
       info "Job.ftp_transfer finished"
+      $queue.counter_inc :jobs_finished
     end
 
-    def notify signal, error = 0, status = {}
-      RestFtpDaemon::Notification.new get(:notify), {
+    def client_notify signal, error = nil, status = {}
+      RestFtpDaemon::Notification.new @notify, {
         id: @id,
         signal: signal,
         error: error,