rest-ftp-daemon 0.85.2 → 0.90.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1c6ed0547c0017afce76bb7b5ffc55e5ddb48981
-  data.tar.gz: 3a40535718c17d96e76a2f332d7e71eef4b3e2ce
+  metadata.gz: e1f8cd7be9c2f0d1932b820c3cf38b7dc79de537
+  data.tar.gz: e001c9fb6c9299f3d1f49b15368fac6628fb0fd9
 SHA512:
-  metadata.gz: 77d115e07981f5bf41d5ef0240e50c813f2a908c5015aff532538a8c56fa9fdf4e2bbb06cbee60e2a51199923f1e38655adbee68d549f28ecdf8866ab2ab6d2b
-  data.tar.gz: 98e520e5b73394dd7f3cb187673b51650290fcd50fb9c19dd7badc89837814571f35550c08aef9e5c4a7de8534858f35e5b08390ea00f5ed0c29406b90f91662
+  metadata.gz: 0f37294ceee1337337cbc4013fd1f8fa08438d5b6ea11fc698cfa1c59442864f4b5cbe7444deedb265e549a5c9a0c32b2e7b4e845c61c1a659656206369f2045
+  data.tar.gz: 28c38f2a5e8618abf1f24773b91381f4623843b9629a190c967641c4980ebc78b9306d6ef26b4bee2da65b6ce616f40f043f5756892b06356b01c9589e15a4f5

data/.gitignore

@@ -4,3 +4,4 @@ pkg
 *.log
 tmp/
 work/
+rest-ftp-daemon.yml

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.84)
+    rest-ftp-daemon (0.88)
       double-bag-ftps
       facter
       grape

data/README.md

@@ -29,6 +29,8 @@ As of today, its main features are :
 * Automatically clean-up jobs after a configurable amount of time (failed, finished)
 * Current bitrate on the last blocks chunk updated in the job attributes
 * Global bitrate on the whole file transfer is re-computed after the transfer finishes
+* Daemon process is tagged with its name and environement in process lists
+* Allow basic patterns in source filename to match multiple files (/dir/file*.jpg)
 
 Expected features in a short-time range :
 
@@ -41,6 +43,9 @@ Expected features in a short-time range :
 * Allow to specify random remote/local source/target
 * Allow more transfer protocols (sFTP, HTTP POST etc)
 
+Known bugs :
+
+* As this project is based on SettingsLogic, which in turns uses Syck YAML parser, configuration merge from "defaults" section and environment-specific section is broken. A sub-tree defined for a specific environment, will overwrite the corresponding subtree from "defaults".
 
 
 Installation
@@ -168,6 +173,17 @@ curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
 '{"source":"~/file.dmg","priority":"3", target":"ftp://anonymous@localhost/incoming/dest4.dmg","notify":"http://requestb.in/1321axg1"}' "http://localhost:3000/jobs"
 ```
 
+
+* Get info about a job with ID="q89j.1"
+
+Both parameters ```q89j.1``` and ```1``` will be accepted as ID in the API. Requests below are equivalent:
+
+```
+GET http://localhost:3100/jobs/q89j.1
+GET http://localhost:3100/jobs/1
+```
+
+
 About
 ------------------------------------------------------------------------------------
 

data/bin/rest-ftp-daemon

@@ -1,6 +1,7 @@
 #!/usr/bin/env ruby
 
-# Try to load libs
+# Try to load libs and constants
+app_root = File.dirname(__FILE__) + '/../'
 begin
   require "thin"
   require 'optparse'
@@ -9,37 +10,24 @@ begin
 rescue LoadError
   raise "EXITING: some of basic libs were not found: thin, optparse, socket, timeout"
 end
+require File.expand_path("#{app_root}/lib/rest-ftp-daemon/constants")
 puts
+#puts "--- #{APP_NAME} #{APP_VER}"
 
-# Detect environment from options
-begin
-  OptionParser.new do |opts|
-    opts.on("-e", "--environment ENV")   { |env| APP_ENV = env }
-    opts.on("", "--dev")                 { APP_ENV = "development" }
-  end.parse(ARGV)
-rescue OptionParser::InvalidOption => e
-end
-
-# Load remaining libs and config subsystem
-app_root = File.dirname(__FILE__) + '/../'
-[:constants, :helpers, :config].each do |lib|
-  require File.expand_path("#{app_root}/lib/rest-ftp-daemon/#{lib.to_s}.rb")
-end
-
-# Setup options and import current ARGV
+# Detect options from ARGV
 options = {}
 parser = OptionParser.new do |opts|
   opts.banner = "Usage: #{File.basename $0} [options] start|stop|restart"
-  opts.on("-e", "--environment ENV", "Environment (#{APP_ENV if defined? APP_ENV})")
-  opts.on("--dev", "Force development environment")
-  opts.on("-p", "--port PORT", "use PORT (#{Settings.port})")          { |port| options["port"] = port.to_i }
-  opts.on("-w", "--workers COUNT", "Use COUNT worker threads (#{Settings['workers']})") { |count| options["workers"] = count.to_i }
-  opts.on("-d", "--daemonize", "Run daemonized in the background")     { |bool| options["daemonize"] = bool || true }
+  opts.on("-e", "--environment ENV")                                   { |env| APP_ENV = env }
+  opts.on("", "--dev")                                                 { APP_ENV = "development" }
+  opts.on("-p", "--port PORT", "use PORT")                             { |port| options["port"] = port.to_i }
+  opts.on("-w", "--workers COUNT", "Use COUNT worker threads")         { |count| options["workers"] = count.to_i }
+  opts.on("-d", "--daemonize", "Run daemonized in the background")     { |bool| options["daemonize"] = bool }
   opts.on("-P", "--pid FILE", "File to store PID")                     { |file| options["pidfile"] = file }
   opts.on("-u", "--user NAME", "User to run daemon as (use with -g)")  { |user| options["user"] = user }
   opts.on("-g", "--group NAME", "Group to run daemon as (use with -u)"){ |group| options["group"] = group }
   opts.on_tail("-h", "--help", "Show this message")                    { puts opts; exit }
-  opts.on_tail('-v', '--version', "Show version")                      { puts Settings['app_ver']; exit }
+  opts.on_tail('-v', '--version', "Show version (#{APP_VER})")                      { puts APP_VER; exit }
 end
 begin
   parser.order!(ARGV)
@@ -52,6 +40,14 @@ rescue OptionParser::InvalidOption => e
   puts "EXITING: option parser: #{e.message}"
   exit 1
 end
+
+# Load helpers and config
+app_root = File.dirname(__FILE__) + '/../'
+[:helpers, :config].each do |lib|
+  require File.expand_path("#{app_root}/lib/rest-ftp-daemon/#{lib.to_s}")
+end
+
+# Merge options from ARGV into settings
 Settings.merge!(options)
 
 # Display compiled configuration
@@ -82,6 +78,7 @@ argv = []
 argv << ["-e", Settings.namespace]
 argv << ["-p", Settings["port"].to_s] unless Settings["port"].nil?
 argv << ["--pid", Settings.pidfile]
+argv << ["--tag", "'#{APP_NAME}/#{Settings.namespace}'"]
 argv << ["--log", Settings["logs"]["thin"].to_s] unless Settings["logs"]["thin"].nil? if Settings["logs"].is_a? Enumerable
 argv << ["--daemonize"] if [1, true].include? Settings["daemonize"]
 # User / group
@@ -94,17 +91,37 @@ puts "--- Thin ARGV"
 puts argv.flatten.join(' ')
 puts
 
-# Start Thin with this rackup configuration
+
+# Start Thin with this rackup configuration, changing to app_root first
 begin
-  Thin::Runner.new(argv.flatten).run!
-  rescue RuntimeError => e
-    puts "FAILED: RuntimeError: #{e.message}"
-  rescue Thin::PidFileExist
-    puts "FAILED: daemon was already running (Thin::PidFileExist)"
-  rescue Thin::PidFileNotFound
-    puts "FAILED: daemon was not running (Thin::PidFileNotFound)"
-  rescue SystemExit
-    # puts "EXITING: daemon in the background (SystemExit)"
-  else
-    # puts "FAILED: process ending"
+  Dir.chdir app_root
+  r = Thin::Runner.new(argv.flatten).run!
+rescue RuntimeError => e
+  puts "FAILED: RuntimeError: #{e.message}"
+rescue Thin::PidFileExist
+  puts "FAILED: daemon was already running (Thin::PidFileExist)"
+rescue Thin::PidFileNotFound
+  puts "FAILED: daemon was not running (Thin::PidFileNotFound)"
+rescue SystemExit
+  # Leave some time for the daemon to launch
+  pidfile = Settings.pidfile
+    print "Waiting for pidfile"
+  while !File.file?(pidfile) do
+    print "."
+    sleep 0.25
+  end
+  puts
+
+  # Check that this process is running
+  pid = File.read(pidfile).to_i
+  begin
+    Process.kill(0, pid)
+    puts "Process ##{pid} is running"
+  rescue Errno::EPERM                     # changed uid
+      puts "No permission to query process ##{pid}!";
+  rescue Errno::ESRCH
+      puts "Process ##{pid} is NOT running.";      # or zombied
+  rescue
+      puts "Unable to determine status for ##{pid}: #{$!}"
+  end
 end

data/lib/rest-ftp-daemon/api/jobs.rb

@@ -28,13 +28,6 @@ module RestFtpDaemon
 
       helpers do
 
-        def next_job_id
-          @@last_worker_id ||= 0
-          @@last_worker_id += 1
-          #Helpers.identifier(IDENT_JOB_BYTES)
-          #SecureRandom.hex(IDENT_JOB_BYTES)
-        end
-
         def threads_with_id job_id
           $threads.list.select do |thread|
             next unless thread[:job].is_a? Job
@@ -45,8 +38,13 @@ module RestFtpDaemon
         def job_describe job_id
           raise RestFtpDaemon::JobNotFound if ($queue.all_size==0)
 
-          # Find job with this id
-          found = $queue.all.select { |job| job.id == job_id }.first
+          # Find job with exactly this id
+          found = $queue.find_by_id(job_id)
+
+          # Find job with this id while searching with the current prefix
+          found = $queue.find_by_id(job_id, true) if found.nil?
+
+          # Check that we did find it
           raise RestFtpDaemon::JobNotFound if found.nil?
           raise RestFtpDaemon::JobNotFound unless found.is_a? Job
 
@@ -71,12 +69,16 @@ module RestFtpDaemon
 
       desc "Get information about a specific job"
       params do
-        requires :id, type: Integer, desc: "job id"
+        requires :id, type: String, desc: "job id", regexp:  /[^\/]+/
+        # optional :audio do
+        #   requires :format, type: Symbol, values: [:mp3, :wav, :aac, :ogg], default: :mp3
+        # end
       end
-      get ':id' do
+      get '*id' do
         info "GET /jobs/#{params[:id]}"
+        #return params
         begin
-          response = job_describe params[:id].to_i
+          response = job_describe params[:id]
         rescue RestFtpDaemon::JobNotFound => exception
           status 404
           api_error exception
@@ -142,12 +144,17 @@ module RestFtpDaemon
           request.body.rewind
           params = JSON.parse(request.body.read, symbolize_names: true)
 
+
+params[:priority] = rand(10)
+
           # Create a new job
           # job_id = $last_worker_id += 1
-          job_id = next_job_id
+          job_id = $queue.generate_id
+          #job = Job.new(params)
           job = Job.new(job_id, params)
 
           # And push it to the queue
+          #$queue.push0 job
           $queue.push job
 
         rescue JSON::ParserError => exception

data/lib/rest-ftp-daemon/api/root.rb

@@ -25,16 +25,6 @@ module RestFtpDaemon
           Root.logger.info(message, level)
         end
 
-        def job_list_by_status
-          statuses = {}
-          alljobs = $queue.all.map do |item|
-            next unless item.is_a? Job
-            statuses[item.get_status] ||= 0
-            statuses[item.get_status] +=1
-          end
-          statuses
-        end
-
       end
 
 
@@ -69,7 +59,11 @@ module RestFtpDaemon
         end
 
         # Jobs to display
-        all_jobs_in_queue = $queue.all
+        # all_jobs_in_queue = $queue.all
+        # all_jobs_in_queue = $queue.ordered_queue
+        # + $queue.popped
+        popped_jobs = $queue.ordered_popped.reverse
+        @jobs_queued = $queue.ordered_queue.reverse
 
         if params["only"].nil? || params["only"].blank?
           @only = nil
@@ -79,19 +73,15 @@ module RestFtpDaemon
 
         case @only
         when nil
-          @jobs = all_jobs_in_queue
-        when :queue
-          @jobs = $queue.queued
+          @jobs_popped = popped_jobs
+        # when :queue
+        #   @jobs_popped = $queue.queued
         else
-          @jobs = $queue.by_status (@only)
+          @jobs_popped = $queue.popped_reverse_sorted_by_status @only
         end
 
         # Count jobs for each status
-        @counts = {}
-        grouped = all_jobs_in_queue.group_by { |job| job.get(:status) }
-        grouped.each do |status, jobs|
-          @counts[status] = jobs.size
-        end
+        @counts = $queue.popped_counts_by_status
 
         # Get workers status
         @gworker_statuses = $pool.get_worker_statuses
@@ -118,7 +108,8 @@ module RestFtpDaemon
           config: Settings.to_hash,
           started: APP_STARTED,
           uptime: (Time.now - APP_STARTED).round(1),
-          status: job_list_by_status,
+          counters: $queue.counters,
+          status: $queue.popped_counts_by_status,
           queue_size: $queue.all_size,
           jobs_queued: $queue.queued.collect(&:id),
           jobs_popped: $queue.popped.collect(&:id),

data/lib/rest-ftp-daemon/constants.rb

@@ -1,19 +1,19 @@
 # Terrific constants
 APP_NAME = "rest-ftp-daemon"
 APP_CONF = "/etc/#{APP_NAME}.yml"
-APP_VER = "0.85.2"
+APP_VER = "0.90.1"
 
 
 # Some global constants
+IDENT_JOB_LEN = 4
 IDENT_NOTIF_LEN = 4
 IDENT_RANDOM_LEN = 8
 
-
 # Some defaults
 DEFAULT_CONNECT_TIMEOUT_SEC = 30
 DEFAULT_UPDATE_EVERY_KB = 2048
 DEFAULT_WORKERS = 1
-DEFAULT_LOGS_PROGNAME_TRIM = 9
+DEFAULT_LOGS_PROGNAME_TRIM = 12
 
 
 # Initialize markers

data/lib/rest-ftp-daemon/exceptions.rb

@@ -9,12 +9,14 @@ module RestFtpDaemon
 
   class JobException              < RestFtpDaemonException; end
   class JobNotFound               < RestFtpDaemonException; end
+  class JobUnresolvedTokens       < RestFtpDaemonException; end
   class JobAssertionFailed        < RestFtpDaemonException; end
   class JobMissingAttribute       < RestFtpDaemonException; end
   class JobSourceNotFound         < RestFtpDaemonException; end
   class JobTargetUnsupported      < RestFtpDaemonException; end
   class JobTargetUnparseable      < RestFtpDaemonException; end
   class JobTargetFileExists       < RestFtpDaemonException; end
+  class JobTargetShouldBeDirectory< RestFtpDaemonException; end
   class JobTooManyOpenFiles       < RestFtpDaemonException; end
 
 end

data/lib/rest-ftp-daemon/helpers.rb

@@ -1,6 +1,5 @@
 require 'securerandom'
 
-
 module RestFtpDaemon
   class Helpers
 
@@ -16,12 +15,20 @@ module RestFtpDaemon
     def self.identifier len
       rand(36**len).to_s(36)
     end
-    #SecureRandom.hex(IDENT_JOB_BYTES)
 
     def self.tokenize(item)
       "[#{item}]"
     end
 
+    def self.contains_tokens(item)
+      /\[[a-zA-Z0-9]+\]/.match(item)
+    end
+
+    def self.extract_filename path
+      m = path.match /\/([^\/]+)$/
+      return m[1] unless m.nil?
+    end
+
     def self.local_port_used? port
       ip = '0.0.0.0'
       timeout = 1
@@ -41,6 +48,23 @@ module RestFtpDaemon
       end
     end
 
+    def self.job_method_label method
+      return if method.nil?
+      klass = case method
+      when :file
+        'label-primary'
+      when :ftp
+        'label-danger'
+      when :ftps
+        'label-warning'
+      when :ftpes
+        'label-warning'
+      else
+         'label-default'
+      end
+      "<div class=\"transfer-method label #{klass}\">#{method.upcase}</div>"
+    end
+
     # def snakecase
     #   gsub(/::/, '/').
     #   gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').

data/lib/rest-ftp-daemon/job.rb

@@ -7,35 +7,36 @@ require 'timeout'
 
 module RestFtpDaemon
   class Job < RestFtpDaemon::Common
+    attr_accessor :id
     attr_accessor :wid
 
-    def initialize(id, params={})
+    def initialize job_id, params={}
       # Call super
       # super()
       info "Job.initialize"
 
-      # Generate new Job.id
-      # $queue.counter_add :transferred, source_size
-
-      # Logger
-      @logger = RestFtpDaemon::Logger.new(:workers, "JOB #{id}")
+      # Init context
+      @params = params
+      @id = job_id.to_s
+      #set :id, job_id
 
       # Protect with a mutex
       @mutex = Mutex.new
 
-      # Init context
-      @params = params
-      set :id, id
+      # Logger
+      @logger = RestFtpDaemon::Logger.new(:workers, "JOB #{id}")
+
+      # Flag current job
       set :started_at, Time.now
       status :created
 
       # Send first notification
-      info "Job.initialize/notify"
+      #info "Job.initialize/notify"
       notify "rftpd.queued"
     end
 
     def id
-      get :id
+      @id
     end
 
     def priority
@@ -61,15 +62,21 @@ module RestFtpDaemon
       rescue RestFtpDaemon::JobSourceNotFound => exception
         return oops "rftpd.started", exception, :job_source_not_found
 
+      rescue RestFtpDaemon::JobUnresolvedTokens => exception
+        return oops "rftpd.started", exception, :job_unresolved_tokens
+
+      rescue RestFtpDaemon::JobTargetUnparseable => exception
+        return oops "rftpd.started", exception, :job_target_unparseable
+
+      rescue RestFtpDaemon::JobAssertionFailed => exception
+        return oops "rftpd.started", exception, :job_assertion_failed
+
       rescue RestFtpDaemon::RestFtpDaemonException => exception
         return oops "rftpd.started", exception, :job_prepare_failed
 
       rescue Exception => exception
         return oops "rftpd.started", exception, :job_prepare_unhandled, true
 
-      rescue exception
-        return oops "rftpd.started", exception, :WOUHOU, true
-
       else
         # Update job's status
         info "Job.process/prepare ok"
@@ -90,21 +97,30 @@ module RestFtpDaemon
         status :starting
         transfer
 
-      rescue Timeout::Error => exception
-        return oops "rftpd.ended", exception, :job_timeout_error
-
-      rescue Net::FTPPermError => exception
-        return oops "rftpd.ended", exception, :job_ftp_perm_error
+      rescue Errno::EHOSTDOWN => exception
+        return oops "rftpd.ended", exception, :job_host_is_down
 
       rescue Errno::ECONNREFUSED => exception
         return oops "rftpd.ended", exception, :job_connexion_refused
 
+      rescue Timeout::Error, Errno::ETIMEDOUT => exception
+        return oops "rftpd.ended", exception, :job_timeout
+
+      rescue Net::FTPPermError => exception
+        return oops "rftpd.ended", exception, :job_perm_error
+
       rescue Errno::EMFILE => exception
         return oops "rftpd.ended", exception, :job_too_many_open_files
 
       rescue RestFtpDaemon::JobTargetFileExists => exception
         return oops "rftpd.ended", exception, :job_target_file_exists
 
+      rescue RestFtpDaemon::JobTargetShouldBeDirectory => exception
+        return oops "rftpd.ended", exception, :job_target_should_be_directory
+
+      rescue RestFtpDaemon::JobAssertionFailed => exception
+        return oops "rftpd.started", exception, :job_assertion_failed
+
       rescue RestFtpDaemon::RestFtpDaemonException => exception
         return oops "rftpd.ended", exception, :job_transfer_failed
 
@@ -124,11 +140,14 @@ module RestFtpDaemon
 
     def describe
       # Update realtime info
-      u = up_time
-      set :uptime, u.round(2) unless u.nil?
+      #u = up_time
+      #set :uptime, u.round(2) unless u.nil?
 
       # Return the whole structure  FIXME
-      @params
+      @params.merge({
+        id: @id,
+        uptime: up_time.round(2)
+        })
       # @mutex.synchronize do
       #   out = @params.clone
       # end
@@ -144,7 +163,7 @@ module RestFtpDaemon
   protected
 
     def up_time
-      return if @params[:started_at].nil?
+      return 0 if @params[:started_at].nil?
       Time.now - @params[:started_at]
     end
 
@@ -175,14 +194,14 @@ module RestFtpDaemon
     end
 
     def expand_path path
-      File.expand_path replace_token(path)
+      File.expand_path replace_tokens(path)
     end
 
     def expand_url path
-      URI::parse replace_token(path) rescue nil
+      URI::parse replace_tokens(path) rescue nil
     end
 
-    def replace_token path
+    def replace_tokens path
       # Ensure endpoints are not a nil value
       return path unless Settings.endpoints.is_a? Enumerable
       vectors = Settings.endpoints.clone
@@ -194,10 +213,14 @@ module RestFtpDaemon
       newpath = path.clone
       vectors.each do |from, to|
         next if to.to_s.blank?
-        #info "Job.replace_token #{Helpers.tokenize(from)} > #{to}"
+        #info "Job.replace_tokens #{Helpers.tokenize(from)} > #{to}"
         newpath.gsub! Helpers.tokenize(from), to
       end
 
+      # Ensure result does not contain tokens after replacement
+      raise RestFtpDaemon::JobUnresolvedTokens if Helpers.contains_tokens newpath
+
+      # All OK, return this
       return newpath
     end
 
@@ -232,18 +255,26 @@ module RestFtpDaemon
       # Check compliance
       raise RestFtpDaemon::JobTargetUnparseable if @target_url.nil?
       raise RestFtpDaemon::JobTargetUnsupported if @target_method.nil?
-      raise RestFtpDaemon::JobSourceNotFound unless File.exists? @source_path
+      #raise RestFtpDaemon::JobSourceNotFound unless File.exists? @source_path
     end
 
     def transfer
-      # Method assertions
-      info "Job.transfer checking_source"
+      # Method assertions and init
       status :checking_source
-      raise RestFtpDaemon::JobAssertionFailed unless @source_path &&  @target_url
+      raise RestFtpDaemon::JobAssertionFailed unless @source_path && @target_url
+      @transfer_sent = 0
+      set :transfer_source_done, 0
 
-      # Init
+      # Guess source file names using Dir.glob
+      source_matches = Dir.glob @source_path
+      info "Job.transfer sources: #{source_matches}"
+      raise RestFtpDaemon::JobSourceNotFound if source_matches.empty?
+      set :transfer_source_count, source_matches.count
+      set :transfer_source_files, source_matches
 
-      target_name = File.basename @target_url.path
+      # Guess target file name, and fail if present while we matched multiple sources
+      target_name = Helpers.extract_filename @target_url.path
+      raise RestFtpDaemon::JobTargetShouldBeDirectory if target_name && source_matches.count>1
 
       # Scheme-aware config
       ftp_init
@@ -251,14 +282,25 @@ module RestFtpDaemon
       # Connect remote server, login and chdir
       ftp_connect
 
-      # Check for target file presence
-      if get(:overwrite).nil? && (ftp_presence target_name)
-        @ftp.close
-        raise RestFtpDaemon::JobTargetFileExists
+      # Check source files presence and compute total size, they should be there, coming from Dir.glob()
+      @transfer_total = 0
+      source_matches.each do |filename|
+        # @ftp.close
+        raise RestFtpDaemon::JobSourceNotFound unless File.exists? filename
+        @transfer_total += File.size filename
+      end
+      set :transfer_total, @transfer_total
+
+      # Handle each source file matched, and start a transfer
+      done = 0
+      source_matches.each do |filename|
+        ftp_transfer filename, target_name
+        done += 1
+        set :transfer_source_done, done
       end
 
-      # Do transfer
-      ftp_transfer target_name
+      # Add total transferred to counter
+      $queue.counter_add :transferred, @transfer_total
 
       # Close FTP connexion
       info "Job.transfer disconnecting"
@@ -320,9 +362,8 @@ module RestFtpDaemon
       info "Job.ftp_connect connect"
       status :ftp_connect
       raise RestFtpDaemon::JobAssertionFailed if @ftp.nil? || @target_url.nil?
-
-        ret = @ftp.connect(@target_url.host)
-        @ftp.passive = true
+      @ftp.connect(@target_url.host)
+      @ftp.passive = true
 
       info "Job.ftp_connect login"
       status :ftp_login
@@ -336,55 +377,76 @@ module RestFtpDaemon
 
     def ftp_presence target_name
       # Method assertions
-      info "Job.ftp_presence"
       status :ftp_presence
       raise RestFtpDaemon::JobAssertionFailed if @ftp.nil? || @target_url.nil?
 
       # Get file list, sometimes the response can be an empty value
       results = @ftp.list(target_name) rescue nil
+      info "Job.ftp_presence: #{results.inspect}"
 
       # Result can be nil or a list of files
       return false if results.nil?
       return results.count >0
     end
 
-    def ftp_transfer target_name
+    def ftp_transfer source_match, target_name = nil
+#target_name
       # Method assertions
-      info "Job.ftp_transfer starting"
+      info "Job.ftp_transfer source_match: #{source_match}"
+      raise RestFtpDaemon::JobAssertionFailed if @ftp.nil?
+      raise RestFtpDaemon::JobAssertionFailed if source_match.nil?
+      #raise RestFtpDaemon::JobAssertionFailed if @transfer_total.nil?
       status :ftp_transfer
-      raise RestFtpDaemon::JobAssertionFailed if @ftp.nil? || @source_path.nil?
+
+      # Use source filename if target path provided none (typically with multiple sources)
+      target_name ||= Helpers.extract_filename source_match
+      info "Job.ftp_transfer target_name: #{target_name}"
+
+      # Check for target file presence
+      status :checking_target
+      overwrite = !get(:overwrite).nil?
+      present = ftp_presence target_name
+      if present
+        if overwrite
+          # delete it first
+          info "Job.ftp_transfer removing target file"
+          @ftp.delete(target_name)
+        else
+          # won't overwrite then stop here
+          info "Job.ftp_transfer failed: target file exists"
+          @ftp.close
+          raise RestFtpDaemon::JobTargetFileExists
+        end
+      end
 
       # Read source file size and parameters
-      source_size = File.size @source_path
-      set :transfer_size, source_size
       update_every_kb = (Settings.transfer.update_every_kb rescue nil) || DEFAULT_UPDATE_EVERY_KB
       notify_after_sec = Settings.transfer.notify_after_sec rescue nil
 
       # Start transfer
-      transferred = 0
       chunk_size = update_every_kb * 1024
       t0 = tstart = Time.now
       notified_at = Time.now
       status :uploading
-      @ftp.putbinaryfile(@source_path, target_name, chunk_size) do |block|
+      @ftp.putbinaryfile(source_match, target_name, chunk_size) do |block|
         # Update counters
-        transferred += block.bytesize
-        set :transfer_sent, transferred
+        @transfer_sent += block.bytesize
+        set :transfer_sent, @transfer_sent
 
         # Update bitrate
-        dt = Time.now - t0
-        bitrate0 = (8 * chunk_size/dt).round(0)
+        #dt = Time.now - t0
+        bitrate0 = get_bitrate(chunk_size, t0).round(0)
         set :transfer_bitrate, bitrate0
 
         # Update job info
-        percent1 = (100.0 * transferred / source_size).round(1)
+        percent1 = (100.0 * @transfer_sent / @transfer_total).round(1)
         set :progress, percent1
 
         # Log progress
         status = []
         status << "#{percent1} %"
-        status << (Helpers.format_bytes transferred, "B")
-        status << (Helpers.format_bytes source_size, "B")
+        status << (Helpers.format_bytes @transfer_sent, "B")
+        status << (Helpers.format_bytes @transfer_total, "B")
         status << (Helpers.format_bytes bitrate0, "bps")
         info "Job.ftp_transfer" + status.map{|txt| ("%#{DEFAULT_LOGS_PROGNAME_TRIM.to_i}s" % txt)}.join("\t")
 
@@ -395,8 +457,8 @@ module RestFtpDaemon
         unless notify_after_sec.nil? || (notified_at + notify_after_sec > Time.now)
           status = {
             progress: percent1,
-            transfer_sent: transferred,
-            transfer_size: source_size,
+            transfer_sent: @transfer_sent,
+            transfer_total: @transfer_total,
             transfer_bitrate: bitrate0
             }
           notify "rftpd.progress", 0, status
@@ -406,12 +468,10 @@ module RestFtpDaemon
       end
 
       # Compute final bitrate
-      tbitrate0 = (8 * source_size.to_f / (Time.now - tstart)).round(0)
+      #tbitrate0 = (8 * @transfer_total.to_f / (Time.now - tstart)).round(0)
+      tbitrate0 = get_bitrate(@transfer_total, tstart).round(0)
       set :transfer_bitrate, tbitrate0
 
-      # Add total transferred to counter
-      $queue.counter_add :transferred, source_size
-
       # Done
       #set :progress, nil
       info "Job.ftp_transfer finished"
@@ -419,12 +479,16 @@ module RestFtpDaemon
 
     def notify signal, error = 0, status = {}
       RestFtpDaemon::Notification.new get(:notify), {
-        id: get(:id),
+        id: @id,
         signal: signal,
         error: error,
         status: status,
         }
     end
 
+    def get_bitrate total, last_timestamp
+      total.to_f / (Time.now - last_timestamp)
+    end
+
   end
 end