rest-ftp-daemon 0.430.1 → 0.432.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 305203e95b7418a3a330d556235fb616abdd7073
-  data.tar.gz: 6cc68e4dd51423492b07efc96293ff9ca04d5f42
+  metadata.gz: 89e094d356f8271d07f64222768a60997ce50a88
+  data.tar.gz: f3deb47de89bc54b1b5f807aa8ecfc868d0c1926
 SHA512:
-  metadata.gz: afa1b54735152ca343dd641a30d819cb3068d4d9010e3b71f9d27c030e87c01bdeab6b00cf655833b9f1e8eda1c419d95a8d914e17ecfb2e1e5372c6f024dd68
-  data.tar.gz: 0d1149d6719f7c08cbcdca104fcfaea357fd18536d67c12eebc0f3b77f5b7f1b1a9c0fcb07cf31b6bc25572694fc219872b77dc7484a1b521827e3b4723b1a5f
+  metadata.gz: ad21099fd7392bc545888394974d95c1e0ce8204e28de1682e3b93087d38c95f20e6594ec59ecc465f562625b9c1def500076aa7a8c3695b1edddea8bd0ff754
+  data.tar.gz: 952ac92bf57af92ec2c99570d5fff30aefcf8b8bf9ba21f3f661b70c8af801757f55ed5c876fc3df590ec8ce708d1ed882bf0ab85cbd6ffbc68ccfc7f397a12a

data/Gemfile.lock

@@ -1,11 +1,11 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.430.1)
+    rest-ftp-daemon (0.432.0)
       activesupport (~> 4.2)
       api-auth
       aws-sdk-resources (~> 2)
-      bmc-daemon-lib (~> 0.3.10)
+      bmc-daemon-lib (~> 0.3.12)
       double-bag-ftps
       facter
       get_process_mem
@@ -39,15 +39,15 @@ GEM
     ast (2.3.0)
     astrolabe (1.3.1)
       parser (~> 2.2)
-    aws-sdk-core (2.5.10)
+    aws-sdk-core (2.5.11)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.5.10)
-      aws-sdk-core (= 2.5.10)
+    aws-sdk-resources (2.5.11)
+      aws-sdk-core (= 2.5.11)
     axiom-types (0.1.1)
       descendants_tracker (~> 0.0.4)
       ice_nine (~> 0.11.0)
       thread_safe (~> 0.3, >= 0.3.1)
-    bmc-daemon-lib (0.3.10)
+    bmc-daemon-lib (0.3.12)
       chamber (~> 2.9)
     builder (3.2.2)
     chamber (2.9.1)

data/README.md

@@ -289,7 +289,7 @@ TODO for this document
 Debian install preparation
 ------------------------------------------------------------------------------------
 
-This project is available as a rubygem, requires Ruby 2.2.2 and RubyGems installed.
+This project is available as a rubygem, requires Ruby 2.3.0 and RubyGems installed.
 
 #### Using rbenv and ruby-build
 
@@ -302,15 +302,15 @@ You may have to install some extra packages for the compilations to complete.
 # git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build
 # echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bashrc
 # echo 'eval "$(rbenv init -)"' >> ~/.bashrc
-# rbenv install --list | grep '2.2'
+# rbenv install --list | grep '2.3'
 ```
 
 ```
 # curl -fsSL https://gist.github.com/mislav/055441129184a1512bb5.txt | rbenv install --patch 2.2.3
 ```
 
-Otherwise, you way have to update ruby-build to include Ruby 2.2 definitions.
-On Debian, 2.2 is not included in Wheezy and appears in Jessie's version of the package.
+Otherwise, you way have to update ruby-build to include Ruby 2.3 definitions.
+On Debian, 2.3 is not included in Wheezy and appears in Jessie's version of the package.
 
 #### Dedicated user
 
@@ -326,8 +326,8 @@ Use a dedicated user for the daemon, switch to this user and enable rbenv
 Install the right ruby version and activate it
 
 ```
-# rbenv install 2.2.2
-# rbenv local 2.2.2
+# rbenv install 2.3.0
+# rbenv local 2.3.0
 # rbenv rehash
 ```
 

data/defaults.yml

@@ -35,8 +35,10 @@ transfer:
   debug_sftp: false
 
   retry_on:                 # job error values that will allow a retry
+    - ftp_temp_error
     - ftp_perm_error
     - net_temp_error
+    - net_perm_error
     - conn_reset_by_peer
     - conn_timed_out
     - conn_refused
@@ -66,16 +68,16 @@ logs:
   path:     "/tmp/"
   level:    debug
 
-  thin:     "rftpd-thin.log"
-  default:  "rftpd-core.log"
+  thin:     "rftpd-default-thin.log"
+  default:  "rftpd-default-core.log"
 
-  queue:    "rftpd-core.log"
-  api:      "rftpd-core.log"
-  workers:  "rftpd-core.log"
-  transfer: "rftpd-workers.log"
-  conchita: "rftpd-workers.log"
-  reporter: "rftpd-workers.log"
-  notify:   "rftpd-workers.log"
+  queue:    "rftpd-default-core.log"
+  api:      "rftpd-default-core.log"
+  workers:  "rftpd-default-core.log"
+  transfer: "rftpd-default-workers.log"
+  conchita: "rftpd-default-workers.log"
+  reporter: "rftpd-default-workers.log"
+  notify:   "rftpd-default-workers.log"
 
-  newrelic: "rftpd-newrelic.log"
-  rollbar:  "rftpd-rollbar.log"
+  newrelic: "rftpd-default-newrelic.log"
+  rollbar:  "rftpd-default-rollbar.log"

data/lib/rest-ftp-daemon/api/jobs.rb

@@ -15,7 +15,7 @@ module RestFtpDaemon
       rescue_from RestFtpDaemon::QueueCantCreateJob do |exception|
         exception_error :api_cant_create_job, 422, exception
       end
-      rescue_from RestFtpDaemon::UnresolvedTokens do |exception|
+      rescue_from RestFtpDaemon::JobUnresolvedTokens do |exception|
         exception_error :api_unresolved_tokens, 422, exception
       end
 

data/lib/rest-ftp-daemon/constants.rb

@@ -101,15 +101,19 @@ WORKER_STATUS_STARTING  = "starting"
 WORKER_STATUS_WAITING   = "waiting"
 WORKER_STATUS_RUNNING   = "running"
 WORKER_STATUS_FINISHED  = "finished"
+WORKER_STATUS_RETRYING  = "retrying"
 WORKER_STATUS_TIMEOUT   = "timeout"
 WORKER_STATUS_CRASHED   = "crashed"
 WORKER_STATUS_CLEANING  = "cleaning"
 WORKER_STATUS_REPORTING = "reporting"
 WORKER_STYLES = {
-  WORKER_STATUS_WAITING  => nil,
-  WORKER_STATUS_RUNNING  => :info,
-  WORKER_STATUS_CRASHED  => :danger,
-  WORKER_STATUS_FINISHED => :success,
+  WORKER_STATUS_WAITING   => nil,
+  WORKER_STATUS_RUNNING   => :info,
+  WORKER_STATUS_REPORTING => :info,
+  WORKER_STATUS_CLEANING  => :info,
+  WORKER_STATUS_RETRYING  => :warning,
+  WORKER_STATUS_CRASHED   => :danger,
+  WORKER_STATUS_FINISHED  => :success,
   }
 
 

data/lib/rest-ftp-daemon/exceptions.rb

@@ -1,31 +1,31 @@
 module RestFtpDaemon
 
-  class RestFtpDaemonException    < StandardError; end
+  class BaseException             < StandardError; end
 
-  class DummyException            < RestFtpDaemonException; end
+  class AssertionFailed           < BaseException; end
 
-  class MissingPool               < RestFtpDaemonException; end
-  class InvalidWorkerNumber       < RestFtpDaemonException; end
-  class QueueCantCreateJob        < RestFtpDaemonException; end
-  class JobException              < RestFtpDaemonException; end
-  class JobTimeout                < RestFtpDaemonException; end
-  class JobNotFound               < RestFtpDaemonException; end
+  class InvalidWorkerNumber       < BaseException; end
+  class QueueCantCreateJob        < BaseException; end
+  class JobTimeout                < BaseException; end
+  class JobNotFound               < BaseException; end
 
+  class JobException              < BaseException; end
+  class JobAttributeMissing       < BaseException; end
+  class JobUnresolvedTokens       < BaseException; end
 
-  class AttributeMissing          < RestFtpDaemonException; end
-  class AssertionFailed           < RestFtpDaemonException; end
-  class UnresolvedTokens          < RestFtpDaemonException; end
-  class LocationMalformed         < RestFtpDaemonException; end
-  class LocationParseError        < RestFtpDaemonException; end
-  class UnsupportedScheme         < RestFtpDaemonException; end
-  class MissingFfmpegLibraries    < RestFtpDaemonException; end
+  class LocationParseError        < BaseException; end
+  class SchemeUnsupported         < BaseException; end
 
-  class SourceNotSupported        < RestFtpDaemonException; end
-  class SourceNotFound            < RestFtpDaemonException; end
+  class SourceUnsupported         < BaseException; end
+  class SourceNotFound            < BaseException; end
+
+  class TargetUnsupported         < BaseException; end
+  class TargetFileExists          < BaseException; end
+  class TargetDirectoryError      < BaseException; end
+  class TargetPermissionError     < BaseException; end
+
+  class VideoMissingBinary        < BaseException; end
+  class VideoMovieError           < BaseException; end
 
-  class TargetNotSupported        < RestFtpDaemonException; end
-  class TargetFileExists          < RestFtpDaemonException; end
-  class TargetDirectoryError      < RestFtpDaemonException; end
-  class TargetPermissionError     < RestFtpDaemonException; end
 
 end

data/lib/rest-ftp-daemon/helpers/views.rb

@@ -23,11 +23,11 @@ module RestFtpDaemon
       "#{MOUNT_JOBS}/#{job.id}" if job.respond_to? :id
     end
 
-    def job_runs_style runs
-      return  "label-outline"   if runs <= 0
-      return  "label-info"      if runs == 1
-      return  "label-warning"   if runs == 2
-      return  "label-danger"    if runs > 2
+    def job_tentatives_style count
+      return  "label-outline"   if count <= 0
+      return  "label-info"      if count == 1
+      return  "label-warning"   if count == 2
+      return  "label-danger"    if count > 2
     end
 
     def location_style uri

data/lib/rest-ftp-daemon/job.rb

@@ -26,7 +26,7 @@ module RestFtpDaemon
     attr_reader :id
     attr_reader :error
     attr_reader :status
-    attr_reader :runs
+    attr_reader :tentatives
 
     attr_reader :queued_at
     attr_reader :updated_at
@@ -55,7 +55,7 @@ module RestFtpDaemon
       @updated_at   = nil
       @error        = nil
       @status       = nil
-      @runs         = 0
+      @tentatives         = 0
       @wid          = nil
 
       # Prepare configuration
@@ -76,11 +76,11 @@ module RestFtpDaemon
       @pool = DEFAULT_POOL unless @pools.keys.include?(@pool)
 
       # Prepare sources/target
-      raise RestFtpDaemon::AttributeMissing, "source" unless params[:source]
+      raise RestFtpDaemon::JobAttributeMissing, "source" unless params[:source]
       @source_loc = Location.new(params[:source])
       log_info "Job.initialize source #{@source_loc.uri}"
 
-      raise RestFtpDaemon::AttributeMissing, "target" unless params[:target]
+      raise RestFtpDaemon::JobAttributeMissing, "target" unless params[:target]
       @target_loc = Location.new(params[:target])
       log_info "Job.initialize target #{@target_loc.uri}"
     end
@@ -89,6 +89,9 @@ module RestFtpDaemon
       # Update job status
       set_status JOB_STATUS_PREPARING
 
+      # Increment run cours
+      @tentatives +=1
+
       # Flag current job timestamps
       @queued_at = Time.now
       @updated_at = Time.now
@@ -101,7 +104,7 @@ module RestFtpDaemon
       set_status JOB_STATUS_QUEUED
       set_error nil
       client_notify :queued
-      log_info "Job.reset notify[queued]"
+      log_info "Job.reset notify[queued] tentative[#{@tentatives}]"
     end
 
     # Process job
@@ -132,7 +135,7 @@ module RestFtpDaemon
       do_after
 
     rescue StandardError => exception
-      Rollbar.error "process: #{exception.class.name}: #{exception.message}"
+      Rollbar.error "Job.process: #{exception.class.name}: #{exception.message}"
       return oops current_signal, exception
 
     else
@@ -159,7 +162,7 @@ module RestFtpDaemon
 
     def weight
       @weight = [
-        - @runs.to_i,
+        - @tentatives.to_i,
         + @priority.to_i,
         - @queued_at.to_i,
         ]

data/lib/rest-ftp-daemon/jobs/errors.rb

@@ -12,12 +12,13 @@ module RestFtpDaemon
       invalid_argument:         Errno::EINVAL,
       runtime_error:            RuntimeError,
 
-      source_not_supported:     RestFtpDaemon::SourceNotSupported,
+      job_timeout:              RestFtpDaemon::JobTimeout,
+      source_not_supported:     RestFtpDaemon::SourceUnsupported,
       source_not_found:         RestFtpDaemon::SourceNotFound,
       target_file_exists:       RestFtpDaemon::TargetFileExists,
       target_directory_error:   RestFtpDaemon::TargetDirectoryError,
       target_permission_error:  RestFtpDaemon::TargetPermissionError,
-      target_not_supported:     RestFtpDaemon::TargetNotSupported,
+      target_not_supported:     RestFtpDaemon::TargetUnsupported,
       assertion_failed:         RestFtpDaemon::AssertionFailed,
       location_parse_error:     RestFtpDaemon::LocationParseError,
 
@@ -58,9 +59,20 @@ module RestFtpDaemon
       sftp_key_mismatch:        Net::SSH::HostKeyMismatch,
       sftp_auth_failed:         Net::SSH::AuthenticationFailed,
       sftp_openssl_error:       OpenSSL::SSL::SSLError,
+
+      video_missing_binary:     RestFtpDaemon::VideoMissingBinary,
+      video_movie_error:        RestFtpDaemon::VideoMovieError,
+
       # rescue Encoding::UndefinedConversionError => exception
       #   return oops :ended, exception, "encoding_error", true
       }
 
+  class InvalidWorkerNumber       < BaseException; end
+  class QueueCantCreateJob        < BaseException; end
+  class JobException              < BaseException; end
+  class JobNotFound               < BaseException; end
+
+
+
   end
 end

data/lib/rest-ftp-daemon/jobs/transfer.rb

@@ -16,26 +16,26 @@ module RestFtpDaemon
       set_info INFO_SOURCE_PROCESSED, 0
 
       # Ensure source is FILE
-      raise RestFtpDaemon::SourceNotSupported, @source_loc.scheme   unless @source_loc.is? URI::FILE
+      raise RestFtpDaemon::SourceUnsupported, @source_loc.scheme   unless @source_loc.is? URI::FILE
 
       # Prepare remote object
       case target_uri
       when URI::FTP
-        log_info "JobTransfer.before target_method FTP"
+        log_info "JobTransfer.do_before target_method FTP"
         @remote = Remote::RemoteFTP.new @target_loc, log_prefix, @config[:debug_ftp]
       when URI::FTPES, URI::FTPS
-        log_info "JobTransfer.before target_method FTPES/FTPS"
+        log_info "JobTransfer.do_before target_method FTPES/FTPS"
         @remote = Remote::RemoteFTP.new @target_loc, log_prefix, @config[:debug_ftps], :ftpes
       when URI::SFTP
-        log_info "JobTransfer.before target_method SFTP"
+        log_info "JobTransfer.do_before target_method SFTP"
         @remote = Remote::RemoteSFTP.new @target_loc, log_prefix, @config[:debug_sftp]
       when URI::S3
-        log_info "JobTransfer.before target_method S3"
+        log_info "JobTransfer.do_before target_method S3"
         @remote = Remote::RemoteS3.new @target_loc, log_prefix, @config[:debug_s3]
       else
         message = "unknown scheme [#{@target_loc.scheme}] [#{target_uri.class.name}]"
-        log_info "JobTransfer.before #{message}"
-        raise RestFtpDaemon::TargetNotSupported, message
+        log_info "JobTransfer.do_before #{message}"
+        raise RestFtpDaemon::TargetUnsupported, message
       end
 
       # Plug this Job into @remote to allow it to log
@@ -51,7 +51,7 @@ module RestFtpDaemon
       sources = @source_loc.scan_files
       set_info INFO_SOURCE_COUNT, sources.size
       set_info INFO_SOURCE_FILES, sources.collect(&:name)
-      log_info "JobTransfer.work sources #{sources.collect(&:name)}"
+      log_info "JobTransfer.do_work sources #{sources.collect(&:name)}"
       raise RestFtpDaemon::SourceNotFound if sources.empty?
 
       # Guess target file name, and fail if present while we matched multiple sources
@@ -95,9 +95,10 @@ module RestFtpDaemon
 
     def do_after
       # Close FTP connexion and free up memory
+      log_info "JobTransfer.do_after close connexion, update status and counters"
       @remote.close
 
-      # Free-up remote object
+      # Free @remote object
       @remote = nil
 
       # Update job status

data/lib/rest-ftp-daemon/jobs/video.rb

@@ -16,16 +16,14 @@ module RestFtpDaemon
       log_info "JobVideo.before target_loc.path: #{@target_loc.path}"
 
       # Ensure FFMPEG lib is available
-      ffmpeg_binary_path = FFMPEG.ffmpeg_binary
-      unless ffmpeg_binary_path && File.exists?(ffmpeg_binary_path)
-        raise RestFtpDaemon::MissingFfmpegLibraries, ffmpeg_binary_path
-      end
+      check_ffmpeg_binary :ffmpeg_binary
+      check_ffmpeg_binary :ffprobe_binary
 
       # Ensure source and target are FILE
-      raise RestFtpDaemon::AssertionFailed                          unless @video_options.is_a? Hash
-      raise RestFtpDaemon::AssertionFailed                          unless @video_custom.is_a? Hash
-      raise RestFtpDaemon::SourceNotSupported, @source_loc.scheme   unless @source_loc.is? URI::FILE
-      raise RestFtpDaemon::TargetNotSupported, @target_loc.scheme   unless @target_loc.is? URI::FILE
+      raise RestFtpDaemon::AssertionFailed                         unless @video_options.is_a? Hash
+      raise RestFtpDaemon::AssertionFailed                         unless @video_custom.is_a? Hash
+      raise RestFtpDaemon::SourceUnsupported, @source_loc.scheme   unless @source_loc.is? URI::FILE
+      raise RestFtpDaemon::TargetUnsupported, @target_loc.scheme   unless @target_loc.is? URI::FILE
     end
 
     def do_work
@@ -57,13 +55,17 @@ module RestFtpDaemon
     end
 
     def ffmpeg_command source, target
-      set_info INFO_SOURCE_CURRENT, source.name
-
       # Read info about source file
-      movie = FFMPEG::Movie.new(source.path)
-      set_info :ffmpeg_size, movie.size
-      set_info :ffmpeg_duration, movie.duration
-      set_info :ffmpeg_resolution, movie.resolution
+      set_info INFO_SOURCE_CURRENT, source.name
+      begin
+        movie = FFMPEG::Movie.new(source.path)
+      rescue StandardError => exception
+        raise RestFtpDaemon::VideoMovieError, exception.message
+      else
+        set_info :ffmpeg_size, movie.size
+        set_info :ffmpeg_duration, movie.duration
+        set_info :ffmpeg_resolution, movie.resolution
+      end
 
       # Build options
       options = {
@@ -101,6 +103,17 @@ module RestFtpDaemon
       return custom_parts
     end
 
+    def check_ffmpeg_binary method
+      # Get or evaluate the path which can raise a Errno::ENOENT
+      path = FFMPEG.send method
+
+      # Check that it returns something which exists on disk
+      raise StandardError unless path && File.exists?(path)
+
+    rescue StandardError, Errno::ENOENT => exception
+      raise RestFtpDaemon::VideoMissingBinary, "missing ffmpeg binary: #{method}"
+    end
+
   end
 end
 

data/lib/rest-ftp-daemon/location.rb

@@ -28,7 +28,7 @@ module RestFtpDaemon
 
     def initialize original
       unless original.is_a? String
-        raise RestFtpDaemon::LocationMalformed, original.inspect
+        raise RestFtpDaemon::AssertionFailed, "location/original/string: #{original.inspect}"
       end
 
       # Strip spaces before/after, copying original "path" at the same time
@@ -42,7 +42,7 @@ module RestFtpDaemon
       # Ensure result does not contain tokens after replacement
       detected_tokens = detect_tokens(location_uri)
       unless detected_tokens.empty?
-        raise RestFtpDaemon::UnresolvedTokens, detected_tokens.join(' ')
+        raise RestFtpDaemon::JobUnresolvedTokens, detected_tokens.join(' ')
       end
 
       # Parse URL and do specific initializations
@@ -54,7 +54,8 @@ module RestFtpDaemon
 
       # Check that scheme is supported
       unless @uri.scheme
-        raise RestFtpDaemon::UnsupportedScheme, url
+        raise RestFtpDaemon::SchemeUnsupported, url
+        # raise RestFtpDaemon::SchemeUnsupported, @uri
       end
     end
 

data/lib/rest-ftp-daemon/remote/base.rb

@@ -26,7 +26,7 @@ module RestFtpDaemon
         @logger = BmcDaemonLib::LoggerPool.instance.get :transfer
 
         # Annnounce object
-        log_info "Remote.initialize debug[#{debug}] target[#{target.path}] "
+        log_info "RemoteBase.initialize debug[#{debug}] target[#{target.path}] "
 
         # Prepare real object
         prepare
@@ -54,6 +54,8 @@ module RestFtpDaemon
       end
 
       def close
+        log_debug "RemoteBase.close"
+
         # Debug mode ?
         return unless @debug
         puts "-------------------- SESSION CLOSING --------------------------"

data/lib/rest-ftp-daemon/remote/ftp.rb

@@ -23,7 +23,7 @@ module RestFtpDaemon
         @chunk_size = JOB_FTP_CHUNKMB.to_i * 1024
 
         # Announce object
-        log_debug "Remote::RemoteFTP.prepare chunk_size:#{@chunk_size}"
+        log_debug "RemoteFTP.prepare chunk_size:#{@chunk_size}"
       end
 
       def connect
@@ -36,7 +36,7 @@ module RestFtpDaemon
 
       def present? target
         size = @ftp.size target.path
-        log_debug "Remote::RemoteFTP.present? [#{target.name}]"
+        log_debug "RemoteFTP.present? [#{target.name}]"
 
       rescue Net::FTPPermError
         return false
@@ -47,13 +47,13 @@ module RestFtpDaemon
       def remove! target
         @ftp.delete target.path
       rescue Net::FTPPermError
-        log_debug "Remote::RemoteFTP.remove! [#{target.name}] not found"
+        log_debug "RemoteFTP.remove! [#{target.name}] not found"
       else
-        log_debug "Remote::RemoteFTP.remove! [#{target.name}] removed"
+        log_debug "RemoteFTP.remove! [#{target.name}] removed"
       end
 
       def mkdir directory
-        log_debug "Remote::RemoteFTP.mkdir [#{directory}]"
+        log_debug "RemoteFTP.mkdir [#{directory}]"
         @ftp.mkdir directory
 
       rescue StandardError => ex
@@ -62,7 +62,7 @@ module RestFtpDaemon
 
       def chdir_or_create directory, mkdir = false
         # Init, extract my parent name and my own name
-        log_debug "Remote::RemoteFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
+        log_debug "RemoteFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
         parent, current = extract_parent(directory)
 
         #dirname, _current = extract_parent(directory)
@@ -98,11 +98,11 @@ module RestFtpDaemon
         end
 
         # Move to the directory
-        log_debug "Remote::RemoteFTP.upload chdir [#{dest.dir}]"
+        log_debug "RemoteFTP.upload chdir [#{dest.dir}]"
         @ftp.chdir "/#{dest.dir}"
 
         # Do the transfer
-        log_debug "Remote::RemoteFTP.upload putbinaryfile [#{dest.name}]"
+        log_debug "RemoteFTP.upload putbinaryfile [#{dest.name}]"
         @ftp.putbinaryfile source.path, dest.name, @chunk_size do |data|
           # Update job status after this block transfer
           yield data.bytesize, dest.name
@@ -110,7 +110,7 @@ module RestFtpDaemon
 
         # Move the file back to its original name
         if use_temp_name
-          log_debug "Remote::RemoteFTP.upload rename [#{dest.name}] > [#{target.name}]"
+          log_debug "RemoteFTP.upload rename [#{dest.name}] > [#{target.name}]"
           @ftp.rename dest.name, target.name
         end
       end

data/lib/rest-ftp-daemon/remote/s3.rb

@@ -13,19 +13,20 @@ module RestFtpDaemon
 
       def prepare
         @multipart_threshold = MULTIPART_THRESHOLD_MB.to_i * 1024 * 1024
-        log_debug "Remote::RemoteS3.prepare target[#{@target.inspect}] #{@multipart_threshold}"
+        log_debug "RemoteS3.prepare target[#{@target.inspect}] #{@multipart_threshold}"
       end
 
       def connect
         super
 
         # Connect init
-        log_debug "Remote::RemoteS3.connect region[#{target.aws_region}] id[#{target.aws_id}]"
+        log_debug "RemoteS3.connect region[#{target.aws_region}] id[#{target.aws_id}]"
 
         # Connect remote server
         @client = Aws::S3::Resource.new(
           region: @target.aws_region,
           credentials: Aws::Credentials.new(@target.aws_id, @target.aws_secret),
+          # thread_count: 4,
           http_wire_trace: @debug
           )
         #s3 = Aws::S3::Client.new(http_wire_trace: true)
@@ -34,7 +35,7 @@ module RestFtpDaemon
       def upload source, target, use_temp_name = false, &callback
         # Push init
         raise RestFtpDaemon::AssertionFailed, "upload/client" if @client.nil?
-        log_debug "Remote::RemoteS3.upload bucket[#{target.aws_bucket}] name[#{target.name}]"
+        log_debug "RemoteS3.upload bucket[#{target.aws_bucket}] name[#{target.name}]"
 
         # Update progress before
         #yield 0, target.name
@@ -63,8 +64,8 @@ module RestFtpDaemon
         #yield target.size, target.name
 
         # Dump information about this file
-        log_debug "Remote::RemoteS3.upload url[#{object.public_url}]"
-        log_debug "Remote::RemoteS3.upload etag[#{object.etag}]"
+        log_debug "RemoteS3.upload url[#{object.public_url}]"
+        log_debug "RemoteS3.upload etag[#{object.etag}]"
         set_info :target_aws_public_url, object.public_url
         set_info :target_aws_etag, object.etag
       end

data/lib/rest-ftp-daemon/remote/sftp.rb

@@ -15,7 +15,7 @@ module RestFtpDaemon
         super
 
         # Connect init
-        log_debug "Remote::RemoteSFTP.connect [#{@target.user}]@[#{@target.host}]:[#{@target.port}]"
+        log_debug "RemoteSFTP.connect [#{@target.user}]@[#{@target.host}]:[#{@target.port}]"
 
         # Debug level
         verbosity =  @debug ? Logger::DEBUG : false
@@ -31,7 +31,7 @@ module RestFtpDaemon
       end
 
       def present? target
-        log_debug "Remote::RemoteSFTP.present? [#{target.name}]"
+        log_debug "RemoteSFTP.present? [#{target.name}]"
         stat = @sftp.stat! target.path
 
       rescue Net::SFTP::StatusException
@@ -41,7 +41,7 @@ module RestFtpDaemon
       end
 
       def remove! target
-        log_debug "Remote::RemoteSFTP.remove! [#{target.name}]"
+        log_debug "RemoteSFTP.remove! [#{target.name}]"
         @sftp.remove target.path
 
       rescue Net::SFTP::StatusException
@@ -51,7 +51,7 @@ module RestFtpDaemon
       end
 
       def mkdir directory
-        log_debug "Remote::RemoteSFTP.mkdir [#{directory}]"
+        log_debug "RemoteSFTP.mkdir [#{directory}]"
         @sftp.mkdir! directory
 
         rescue StandardError => ex
@@ -60,7 +60,7 @@ module RestFtpDaemon
 
       def chdir_or_create directory, mkdir = false
         # Init, extract my parent name and my own name
-        log_debug "Remote::RemoteSFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
+        log_debug "RemoteSFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
         parent, _current = extract_parent(directory)
 
         # Access this directory
@@ -99,7 +99,7 @@ module RestFtpDaemon
         end
 
         # Do the transfer
-        log_debug "Remote::RemoteSFTP.upload temp[#{use_temp_name}] name[#{dest.name}]"
+        log_debug "RemoteSFTP.upload temp[#{use_temp_name}] name[#{dest.name}]"
         @sftp.upload! source.path, dest.path do |event, _uploader, *args|
           case event
           when :open then
@@ -125,7 +125,7 @@ module RestFtpDaemon
         # Move the file back to its original name
         if use_temp_name
           flags = 0x00000001
-          log_debug "Remote::RemoteSFTP.upload rename [#{dest.name}] > [#{target.name}]"
+          log_debug "RemoteSFTP.upload rename [#{dest.name}] > [#{target.name}]"
           @sftp.rename! dest.path, target.path, flags
         end
 

data/lib/rest-ftp-daemon/views/dashboard_jobs.haml

@@ -42,7 +42,7 @@
           %th.text-right{title: "updated every #{JOB_FTP_CHUNKMB} MB block"} rate
           %th{title: "Worker ID"} W
           %th{title: "Priority"} P
-          %th{title: "Runs count"} R
+          %th{title: "Tentatives count"} T
 
         - @jobs_queued.each do |pool, jobs|
           - unless jobs.empty?

data/lib/rest-ftp-daemon/views/dashboard_table.haml

@@ -13,7 +13,6 @@
   - job_working       = [JOB_STATUS_UPLOADING, JOB_STATUS_TRANSFORMING].include? job.status
 
   - trclass = JOB_STYLES[job.status]
-  - runs = job.runs.to_i
 
   - unless job.error.nil?
     - trclass = "warning"
@@ -84,4 +83,4 @@
         .label.label-default.flag.worker-label= job.priority
 
     %td
-      .label.flag.worker-label{class: job_runs_style(runs)}= runs
+      .label.flag.worker-label{class: job_tentatives_style(job.tentatives)}= job.tentatives

data/lib/rest-ftp-daemon/workers/reporter.rb

@@ -10,7 +10,7 @@ module RestFtpDaemon
       config_section :reporter
 
       # Other configuration options
-      @report_newrelic = Conf.feature?(:newrelic)
+      @feature_newrelic = Conf.feature?(:newrelic)
 
       # Check that everything is OK
       return "invalid timer"      unless @config[:timer].to_i > 0
@@ -34,9 +34,6 @@ module RestFtpDaemon
   private
 
     def do_metrics
-      # What metrics to report?
-      report_newrelic = Conf.feature?(:newrelic)
-
       # Get common metrics and dump them to logs
       log_debug "begin metrics sample"
       metrics = Metrics.sample
@@ -46,10 +43,10 @@ module RestFtpDaemon
         log_error "unable to collect metrics"
         return
       end
-      log_info "collected metrics (newrelic: #{@report_newrelic})", metrics
+      log_info "collected metrics (newrelic: #{@feature_newrelic.inspect})", metrics
 
       # Transpose metrics to NewRelic metrics
-      report_newrelic(metrics) if @report_newrelic
+      report_newrelic(metrics) if @feature_newrelic
     end
 
     def report_newrelic metrics

data/lib/rest-ftp-daemon/workers/transfer.rb

@@ -56,20 +56,17 @@ module RestFtpDaemon
         job.process
       end
 
-      # Processing done
-      worker_status WORKER_STATUS_FINISHED, job
-
       # Increment total processed jobs count
       RestFtpDaemon::Counters.instance.increment :jobs, :processed
 
     rescue RestFtpDaemon::JobTimeout => ex
-      log_error "JOB TIMED OUT", ex.backtrace
-      worker_status WORKER_STATUS_TIMEOUT
+      log_error "JOB TIMEOUT", ex.backtrace
+      worker_status WORKER_STATUS_TIMEOUT, job
 
       # Inform the job
       job.oops_you_stop_now ex unless job.nil?
 
-    rescue RestFtpDaemon::AssertionFailed, RestFtpDaemon::AttributeMissing, StandardError => ex
+    rescue RestFtpDaemon::AssertionFailed, RestFtpDaemon::JobAttributeMissing, StandardError => ex
       log_error "JOB EXCEPTION ex[#{ex.class}] #{ex.message}", ex.backtrace
       worker_status WORKER_STATUS_CRASHED
 
@@ -80,30 +77,61 @@ module RestFtpDaemon
     def handle_job_result job
       # If job status requires a retry, just restack it
       if !job.error
-        #log_info "job succeeded"
+        # Processing successful
+        log_error "job finished with no error"
+        worker_status WORKER_STATUS_FINISHED, job
 
-      elsif !(@config[:retry_on].is_a?(Enumerable) && @config[:retry_on].include?(job.error))
-        log_error "not retrying: error not eligible"
+      elsif error_not_eligible(job)
+        log_error "not retrying [#{job.error}] retry_on not eligible"
 
-      elsif @config[:retry_for] && (job.age >= @config[:retry_for])
-        log_error "not retrying: max_age reached (#{@config[:retry_for]} s)"
+      elsif error_reached_for(job)
+        log_error "not retrying [#{job.error}] retry_for reached [#{@config[:retry_for]}s]"
 
-      elsif @config[:retry_max] && (job.runs >= @config[:retry_max])
-        log_error "not retrying: max_runs reached (#{@config[:retry_max]} tries)"
+      elsif error_reached_max(job)
+        log_error "not retrying [#{job.error}] retry_max reached #{tentatives(job)}"
 
       else
         # Delay cannot be negative, and will be 1s minimum
         retry_after = [@config[:retry_after] || DEFAULT_RETRY_AFTER, 1].max
-        log_info "retrying job: waiting for #{retry_after} seconds"
+        log_info "retry job [#{job.id}] in [#{retry_after}s] tried #{tentatives(job)}"
 
         # Wait !
+        worker_status WORKER_STATUS_RETRYING, job
         sleep retry_after
-        log_info "retrying job: requeued after delay"
+        log_debug "job [#{job.id}] requeued after [#{retry_after}s] delay"
 
         # Now, requeue this job
         RestFtpDaemon::JobQueue.instance.requeue job
       end
     end
 
+    def error_not_eligible job
+      # No, if no eligible errors
+      return true unless @config[:retry_on].is_a?(Enumerable)
+
+      # Tell if this error is in the list
+      return !@config[:retry_on].include?(job.error.to_s)
+    end
+
+    def error_reached_for job
+      # Not above, if no limit definded
+      return false unless @config[:retry_for]
+
+      # Job age above this limit
+      return job.age >= @config[:retry_for]
+    end
+
+    def error_reached_max job
+      # Not above, if no limit definded
+      return false unless @config[:retry_max]
+
+      # Job age above this limit
+      return job.tentatives >= @config[:retry_max]
+    end
+
+    def tentatives job
+      "[#{job.tentatives}/#{@config[:retry_max]}]"
+    end
+
   end
 end

data/rest-ftp-daemon.gemspec

@@ -2,7 +2,7 @@
 Gem::Specification.new do |spec|
 
   # Project version
-  spec.version                    = "0.430.1"
+  spec.version                    = "0.432.0"
 
   # Project description
   spec.name                       = "rest-ftp-daemon"
@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
                                       reject{ |f| f =~ /^dashboard.+\.png/ }
   spec.executables                = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths              = ["lib"]
-  spec.required_ruby_version      = ">= 2.2.2"
+  spec.required_ruby_version      = ">= 2.3"
 
   # Development dependencies
   spec.add_development_dependency "bundler", "~> 1.6"
@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "http"
 
   # Runtime dependencies
-  spec.add_runtime_dependency     "bmc-daemon-lib", "~> 0.3.10"
+  spec.add_runtime_dependency     "bmc-daemon-lib", "~> 0.3.12"
   spec.add_runtime_dependency     "json", "~> 1.8"
   spec.add_runtime_dependency     "thin", "~> 1.7"
   spec.add_runtime_dependency     "activesupport", "~> 4.2"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: 0.430.1
+  version: 0.432.0
 platform: ruby
 authors:
 - Bruno MEDICI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-08 00:00:00.000000000 Z
+date: 2016-09-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -100,14 +100,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.10
+        version: 0.3.12
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.10
+        version: 0.3.12
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -524,7 +524,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.2.2
+      version: '2.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -532,7 +532,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.6.6
 signing_key: 
 specification_version: 4
 summary: RESTful FTP client daemon