rest-ftp-daemon 0.245 → 0.245.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c9be0d61a767998e3daba2c1006ba9c8b184e303
-  data.tar.gz: 46b7e9173008c7f5e4cb015ae84b1d20ac788cd7
+  metadata.gz: 543ad77656660d81d07cf854694e58db9c31fb3b
+  data.tar.gz: 8562831e868f6215c41d58096f593d49265887d9
 SHA512:
-  metadata.gz: fec8c694546fc1d7b2991f2d0d8e1fa2bf35669a7e6cd56dffd459b787db66ab9a608205635dce427720022b7c3b94aaade650c110674042d2cb302d0def058f
-  data.tar.gz: 45621e4a9a99a535c5448a5172986d391f76f396feab220efb89979ee1f0be63499b5edd1ca8012b8cae275ae7b9dff7c5997ee67c638b8d7db541219a29b528
+  metadata.gz: 06381f06990e68bfe3a1c4091291c91bb0fed0f3e98e12b60123b11e3b529c2406c301c4a3f450d5121447a7bde0d6a2a7763338323a453eff23d0396f50fd0c
+  data.tar.gz: 962fba554f02dcb527fbfe9d27cf369bcd196feb374a27a4df872063a3f93bc5517bf81d6caedc2dd0cf7c20d00ca702392e480fb6ba8aa45be9075d3adaad02

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.245)
+    rest-ftp-daemon (0.245.1)
       double-bag-ftps
       facter
       get_process_mem

data/README.md

@@ -21,7 +21,7 @@ Features
   * environment-aware configuration in a YAML file
   * daemon process is tagged with its name and environment in process lists
   * global dashboard directly served within the daemon HTTP interface
-
+  * support pooling of worker to separate capacity among jobs
 
 * File management ans transferts
   * allow authentication in FTP target in a standard URI-format
@@ -97,7 +97,6 @@ Launcher options :
 | -p      | --port        | (automatic)   | Port to listen for API requests                             |
 | -e      |               | production    | Environment name                                            |
 |         | --dev         |               | Equivalent to -e development                                |
-| -w      | --workers     | 1             | Number of workers spawned at launch                         |
 | -d      | --daemonize   | false         | Wether to send the daemon to background                     |
 | -f      | --foreground  | false         | Wether to keep the daemon running in the shell              |
 | -P      | --pid         | (automatic)   | Path of the file containing the PID                         |
@@ -140,6 +139,15 @@ curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
 ```
 
 
+#### Start a job with a specific pool name
+
+```
+curl -H "Content-Type: application/json" -X POST -D /dev/stdout -d \
+'{"pool": "maxxxxx",source":"~/file.iso",target":"ftp://anonymous@localhost/incoming/dest2.iso"}' "http://localhost:3000/jobs"
+```
+This job will be handled by the "maxxxxx" workers only, or by the default worker is this pool is not declared.
+
+
 #### Get info about a job with ID="q89j.1"
 
 Both parameters `q89j.1` and `1` will be accepted as ID in the API. Requests below are equivalent:

data/bin/rest-ftp-daemon

@@ -28,7 +28,6 @@ parser = OptionParser.new do |opts|
   opts.on("-e", "--environment ENV")                                    { |env| APP_ENV = env }
   opts.on("",   "--dev")                                                { APP_ENV = "development" }
   opts.on("-p", "--port PORT", "use PORT")                              { |port| options["port"] = port.to_i }
-  opts.on("-w", "--workers COUNT", "Use COUNT worker threads")          { |count| options["workers"] = count.to_i }
   opts.on("-d", "--daemonize", "Run daemonized in the background")      { options["daemonize"] = true }
   opts.on("-f", "--foreground", "Run in the foreground")                { options["daemonize"] = false }
   opts.on("-P", "--pid FILE", "File to store PID")                      { |file| options["pidfile"] = file }

data/lib/rest-ftp-daemon/constants.rb

@@ -1,7 +1,7 @@
 # Terrific constants
 APP_NAME = "rest-ftp-daemon"
 APP_NICK = "rftpd"
-APP_VER = "0.245"
+APP_VER = "0.245.1"
 
 # Provide default config file information
 APP_LIB = File.expand_path File.dirname(__FILE__)
@@ -22,7 +22,7 @@ EOD
 # Configuration defaults
 # DEFAULT_WORKERS         = 2
 DEFAULT_POOL            = "default"
-DEFAULT_WORKER_TIMEOUT  = 3600  # 1h
+DEFAULT_WORKER_TIMEOUT  = 1800  # 1h
 DEFAULT_SFTP_TIMEOUT    = 600   # 10mn
 DEFAULT_FTP_CHUNK       = 1024  # 1 MB
 DEFAULT_PAGE_SIZE       = 50    # 50 lines
@@ -37,26 +37,26 @@ JOB_UPDATE_INTERVAL     = 1
 
 
 # Jobs and workers statuses
-JOB_STATUS_PREPARING    = :preparing
-JOB_STATUS_RUNNING      = :running
-JOB_STATUS_CHECKING_SRC = :checking_source
-JOB_STATUS_CONNECTING   = :remote_connect
-JOB_STATUS_CHDIR        = :remote_chdir
-JOB_STATUS_UPLOADING    = :uploading
-JOB_STATUS_RENAMING     = :renaming
-JOB_STATUS_PREPARED     = :prepared
-JOB_STATUS_DISCONNECTING= :remote_disconnect
-JOB_STATUS_FINISHED     = :finished
-JOB_STATUS_FAILED       = :failed
-JOB_STATUS_QUEUED       = :queued
-
-WORKER_STATUS_STARTING  = :starting
-WORKER_STATUS_WAITING   = :waiting
-WORKER_STATUS_RUNNING   = :running
-WORKER_STATUS_FINISHED  = :finished
-WORKER_STATUS_TIMEOUT   = :timeout
-WORKER_STATUS_CRASHED   = :crashed
-WORKER_STATUS_CLEANING  = :cleaning
+JOB_STATUS_PREPARING    = "preparing"
+JOB_STATUS_RUNNING      = "running"
+JOB_STATUS_CHECKING_SRC = "checking_source"
+JOB_STATUS_CONNECTING   = "remote_connect"
+JOB_STATUS_CHDIR        = "remote_chdir"
+JOB_STATUS_UPLOADING    = "uploading"
+JOB_STATUS_RENAMING     = "renaming"
+JOB_STATUS_PREPARED     = "prepared"
+JOB_STATUS_DISCONNECTING= "remote_disconnect"
+JOB_STATUS_FINISHED     = "finished"
+JOB_STATUS_FAILED       = "failed"
+JOB_STATUS_QUEUED       = "queued"
+
+WORKER_STATUS_STARTING  = "starting"
+WORKER_STATUS_WAITING   = "waiting"
+WORKER_STATUS_RUNNING   = "running"
+WORKER_STATUS_FINISHED  = "finished"
+WORKER_STATUS_TIMEOUT   = "timeout"
+WORKER_STATUS_CRASHED   = "crashed"
+WORKER_STATUS_CLEANING  = "cleaning"
 
 
 # Logging and startup
@@ -84,18 +84,17 @@ NOTIFY_IDENTIFIER_LEN   = 4
 
 # Dashboard row styles
 DASHBOARD_JOB_STYLES = {
-  JOB_STATUS_QUEUED     => :active,
-  JOB_STATUS_FAILED     => :warning,
-  JOB_STATUS_FINISHED   => :success,
-  JOB_STATUS_UPLOADING  => :info,
-  JOB_STATUS_RENAMING   => :info,
+  JOB_STATUS_QUEUED      => :active,
+  JOB_STATUS_FAILED      => :warning,
+  JOB_STATUS_FINISHED    => :success,
+  JOB_STATUS_UPLOADING   => :info,
+  JOB_STATUS_RENAMING    => :info,
   }
 DASHBOARD_WORKER_STYLES = {
-  waiting:              :success,
-  running:              :info,
-  crashed:              :danger,
-  done:                 :success,
-  dead:                 :danger,
+  WORKER_STATUS_WAITING  => :success,
+  WORKER_STATUS_RUNNING  => :info,
+  WORKER_STATUS_CRASHED  => :danger,
+  WORKER_STATUS_FINISHED => :success,
   }
 
 

data/lib/rest-ftp-daemon/job.rb

@@ -104,22 +104,22 @@ module RestFtpDaemon
         prepare
 
       rescue RestFtpDaemon::JobMissingAttribute => exception
-        return oops :started, exception, :missing_attribute
+        return oops :started, exception, "missing_attribute"
 
       rescue RestFtpDaemon::JobUnresolvedTokens => exception
-        return oops :started, exception, :unresolved_tokens
+        return oops :started, exception, "unresolved_tokens"
 
       rescue RestFtpDaemon::JobTargetUnparseable => exception
-        return oops :started, exception, :target_unparseable
+        return oops :started, exception, "target_unparseable"
 
       rescue RestFtpDaemon::JobTargetUnsupported => exception
-        return oops :started, exception, :target_unsupported
+        return oops :started, exception, "target_unsupported"
 
       rescue RestFtpDaemon::JobAssertionFailed => exception
-        return oops :started, exception, :assertion_failed
+        return oops :started, exception, "assertion_failed"
 
       rescue URI::InvalidURIError => exception
-        return oops :started, exception, :target_invalid
+        return oops :started, exception, "target_invalid"
 
       else
         # Prepare done !
@@ -130,77 +130,77 @@ module RestFtpDaemon
 
       # Process job
       begin
-        #raise Net::FTPTempError, '451 Téléchargement avorté. Input/output error'.force_encoding("ASCII-8BIT")
+        #return oops :ended, Exception.new, "ftp_perm_error"
         run
 
       rescue SocketError => exception
-        return oops :ended, exception, :conn_socket_error
+        return oops :ended, exception, "conn_socket_error"
 
       rescue EOFError => exception
-        return oops :ended, exception, :conn_eof
+        return oops :ended, exception, "conn_eof"
 
       rescue Errno::EHOSTDOWN => exception
-        return oops :ended, exception, :conn_host_is_down
+        return oops :ended, exception, "conn_host_is_down"
 
       rescue Errno::ENETUNREACH => exception
-        return oops :ended, exception, :conn_unreachable
+        return oops :ended, exception, "conn_unreachable"
 
       rescue Errno::ECONNRESET => exception
-        return oops :ended, exception, :conn_reset_by_peer
+        return oops :ended, exception, "conn_reset_by_peer"
 
       rescue Errno::ENOTCONN => exception
-        return oops :ended, exception, :conn_failed
+        return oops :ended, exception, "conn_failed"
 
       rescue Errno::ECONNREFUSED => exception
-        return oops :ended, exception, :conn_refused
+        return oops :ended, exception, "conn_refused"
 
       rescue Timeout::Error, Errno::ETIMEDOUT, Net::ReadTimeout => exception
-        return oops :ended, exception, :conn_timed_out
+        return oops :ended, exception, "conn_timed_out"
 
       rescue OpenSSL::SSL::SSLError => exception
-        return oops :ended, exception, :conn_openssl_error
+        return oops :ended, exception, "conn_openssl_error"
 
       rescue Net::FTPPermError => exception
-        return oops :ended, exception, :ftp_perm_error
+        return oops :ended, exception, "ftp_perm_error"
 
       rescue Net::FTPTempError => exception
-        return oops :ended, exception, :net_temp_error
+        return oops :ended, exception, "net_temp_error"
 
       rescue Net::SFTP::StatusException => exception
-        return oops :ended, exception, :sftp_exception
+        return oops :ended, exception, "sftp_exception"
 
       rescue Net::SSH::HostKeyMismatch => exception
-        return oops :ended, exception, :sftp_key_mismatch
+        return oops :ended, exception, "sftp_key_mismatch"
 
       rescue Net::SSH::AuthenticationFailed => exception
-        return oops :ended, exception, :sftp_auth_failed
+        return oops :ended, exception, "sftp_auth_failed"
 
       rescue Errno::EMFILE => exception
-        return oops :ended, exception, :too_many_open_files
+        return oops :ended, exception, "too_many_open_files"
 
       rescue Errno::EINVAL => exception
-        return oops :ended, exception, :invalid_argument, true
+        return oops :ended, exception, "invalid_argument", true
 
       rescue Encoding::UndefinedConversionError => exception
-        return oops :ended, exception, :encoding_error, true
+        return oops :ended, exception, "encoding_error", true
 
       rescue RestFtpDaemon::JobSourceNotFound => exception
-        return oops :ended, exception, :source_not_found
+        return oops :ended, exception, "source_not_found"
 
       rescue RestFtpDaemon::JobSourceNotReadable => exception
-        return oops :ended, exception, :source_not_readable
+        return oops :ended, exception, "source_not_readable"
 
       rescue RestFtpDaemon::JobTargetFileExists => exception
-        return oops :ended, exception, :target_file_exists
+        return oops :ended, exception, "target_file_exists"
 
       rescue RestFtpDaemon::JobTargetDirectoryError => exception
-        return oops :ended, exception, :target_directory_missing
+        return oops :ended, exception, "target_directory_missing"
 
       rescue RestFtpDaemon::JobTargetPermissionError => exception
-        return oops :ended, exception, :target_permission_error
+        return oops :ended, exception, "target_permission_error"
 
       rescue RestFtpDaemon::JobAssertionFailed => exception
-        return oops :ended, exception, :assertion_failed
+        return oops :ended, exception, "assertion_failed"
 
       else
         # All done !
@@ -231,11 +231,11 @@ module RestFtpDaemon
     end
 
     def oops_after_crash exception
-      oops :ended, exception, :crashed
+      oops :ended, exception, "crashed"
     end
 
     def oops_you_stop_now exception
-      oops :ended, exception, :timeout
+      oops :ended, exception, "timeout"
     end
 
     def age

data/lib/rest-ftp-daemon/worker_job.rb

@@ -35,6 +35,8 @@ module RestFtpDaemon
       max_age = Settings.at(:retry, :max_age)
       max_runs = Settings.at(:retry, :max_runs)
       delay = Settings.at(:retry, :delay)
+      # log_error "on_errors: #{on_errors.inspect} (#{on_errors.first.class})"
+      # log_error "job.error: #{job.error} (#{job.error.class})"
 
       if !job.error
         #log_info "job succeeded"

data/rest-ftp-daemon.yml.sample

@@ -1,11 +1,13 @@
 defaults: &defaults
   daemonize: true
   port: 3000
-  workers: 2
   user: rftpd
   group: rftpd
   #host: "myhost"
 
+  pools:
+    default: 2
+
   transfer:
     # notify_after_sec: 5       # wait at least X seconds between HTTP notifications
     # mkdir: true               # build directory tree if missing
@@ -14,7 +16,17 @@ defaults: &defaults
     # timeout: 1800             # jobs running for longer than X seconds will be killed
 
   retry:
-    # on_errors: ["net_temp_error", "conn_reset_by_peer", "conn_timed_out", "conn_refused", "sftp_auth_failed", "conn_host_is_down", "conn_unreachable", "conn_failed", "conn_openssl_error"]
+    # on_errors:
+    #   - ftp_perm_error
+    #   - net_temp_error
+    #   - conn_reset_by_peer
+    #   - conn_timed_out
+    #   - conn_refused
+    #   - sftp_auth_failed
+    #   - conn_host_is_down
+    #   - conn_unreachable
+    #   - conn_failed
+    #   - conn_openssl_error
     # max_runs: 5
     # max_age: 1800
     # delay: 10

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: '0.245'
+  version: 0.245.1
 platform: ruby
 authors:
 - Bruno MEDICI