rest-ftp-daemon 0.231.1 → 0.240.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b0b3836e8b886599053c67f24487187570d24f7f
-  data.tar.gz: 678679b81b46d4cb3282c3ed28a1f668e82bb127
+  metadata.gz: d93647c6c82dc619334ae8f906604f0b3472def0
+  data.tar.gz: 00175925d0ffe1d63c3f8b6f21a79985f02ebbb5
 SHA512:
-  metadata.gz: e5f5dad39c56dae2b4a08b720851d884375ecbf0ba8e43e33fce0f700d629b1831102ec1d20658513559d38491a7debb15b9352e32f2cc0e306d5927c8d17ae7
-  data.tar.gz: 724fcc5d35cbd4b6190490b88403b78f56ba6a904a63468991cb8ad9937b62ca21fd9ad7c9b4b07cbab0b1025ff5e81d41cf508be70826395fb5734ed0af02bb
+  metadata.gz: 08ec5e3fe94b08fd1caa9cae780e32a5ddff96736ce8dcee10d0da70356b36adc3b60bee52c825eac97f3a1704c4aab450c7d778b710eb42477ac2c74ef5defc
+  data.tar.gz: 3479a4a57c9396f39bf8cb5a50b061a1887b5bd2e565621f856976dc7b55b4a51a0c4b078cde8e216c41e81aa91f40cae6b0ee10091f550a725281530012c43f

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.231.1)
+    rest-ftp-daemon (0.240.0)
       double-bag-ftps
       facter
       get_process_mem
@@ -10,6 +10,7 @@ PATH
       haml
       json
       net-sftp
+      net-ssh (~> 2.10.1.rc1)
       newrelic_rpm
       settingslogic
       sys-cpu
@@ -84,7 +85,7 @@ GEM
     multi_xml (0.5.5)
     net-sftp (2.1.2)
       net-ssh (>= 2.6.5)
-    net-ssh (2.9.2)
+    net-ssh (2.10.1.rc1)
     newrelic_rpm (3.13.0.299)
     parser (2.2.2.6)
       ast (>= 1.1, < 3.0)

data/bin/rest-ftp-daemon

@@ -45,7 +45,6 @@ parser = OptionParser.new do |opts|
 end
 
 
-
 # Parse options and check compliance
 begin
   parser.order!(ARGV)
@@ -60,6 +59,7 @@ end
 # Load config, and merge options from ARGV into settings
 # FIXME: file configuration detection could reside in settings.rb
 APP_CONF ||= DEFAULT_CONFIG_PATH
+APP_ENV  ||= "production"
 abort "EXITING: cannot read configuration file: #{APP_CONF}" unless File.exist? APP_CONF
 begin
   # Import settings
@@ -92,12 +92,13 @@ end
 
 
 # ARGV: build final
+thin_logs = Settings.at(:logs, :thin)
 argv = []
 argv << ["-e", Settings.namespace]
 argv << ["-p", Settings.port.to_s] if Settings.port
 argv << ["--pid", Settings.pidfile]
 argv << ["--tag", "'#{APP_NAME}/#{Settings.namespace}'"]
-argv << ["--log", Settings.at(:logs, :thin)]
+argv << ["--log", thin_logs] unless !thin_logs
 argv << ["--daemonize"] if [1, true].include? Settings.daemonize
 
 

data/lib/rest-ftp-daemon/api/root.rb

@@ -29,7 +29,7 @@ module RestFtpDaemon
 
         # Check that Queue and Pool are available
         raise RestFtpDaemon::MissingQueue unless defined? $queue
-        raise RestFtpDaemon::MissingQueue unless defined? $pool
+        raise RestFtpDaemon::MissingPool unless defined? $pool
       end
 
 
@@ -63,7 +63,7 @@ module RestFtpDaemon
 
         status 200
         return  {
-          hostname: `hostname`.chomp,
+          hostname: `hostname`.to_s.chomp,
           version: APP_VER,
           started: APP_STARTED,
           uptime: (Time.now - APP_STARTED).round(1),
@@ -88,6 +88,19 @@ module RestFtpDaemon
         return Helpers.get_censored_config
       end
 
+      # Server config
+      post "/config/reload" do
+        log_info "POST /config/reload"
+
+        if Settings.at(:debug, :allow_reload)==true
+          Settings.reload!
+          status 200
+          return Helpers.get_censored_config
+        else
+          status 403
+          return "Config reload not permitted"
+        end
+      end
 
     end
   end

data/lib/rest-ftp-daemon/constants.rb

@@ -1,7 +1,7 @@
 # Terrific constants
 APP_NAME = "rest-ftp-daemon"
 APP_NICK = "rftpd"
-APP_VER = "0.231.1"
+APP_VER = "0.240.0"
 
 # Provide default config file information
 APP_LIB = File.expand_path File.dirname(__FILE__)

data/lib/rest-ftp-daemon/job.rb

@@ -16,6 +16,7 @@ module RestFtpDaemon
     attr_reader :id
     attr_reader :error
     attr_reader :status
+    attr_reader :runs
 
     attr_reader :queued_at
     attr_reader :updated_at
@@ -40,6 +41,7 @@ module RestFtpDaemon
       @finished_at = nil
       @error = nil
       @status = nil
+      @runs = 0
       @wid = nil
 
       # Logger
@@ -146,6 +148,9 @@ module RestFtpDaemon
       rescue Net::SFTP::StatusException => exception
         return oops :ended, exception, :sftp_exception
 
+      rescue Net::SSH::AuthenticationFailed => exception
+        return oops :ended, exception, :sftp_auth_failed
+
       rescue Errno::EMFILE => exception
         return oops :ended, exception, :too_many_open_files
 
@@ -186,7 +191,11 @@ module RestFtpDaemon
     end
 
     def weight
-      @weight = [@priority.to_i, -@queued_at.to_i]
+      @weight = [
+        - @runs.to_i,
+        + @priority.to_i,
+        - @queued_at.to_i
+        ]
     end
 
     def exectime
@@ -207,13 +216,13 @@ module RestFtpDaemon
       oops :ended, exception, :timeout
     end
 
-  protected
-
     def age
       return nil if @queued_at.nil?
       (Time.now - @queued_at).round(2)
     end
 
+  protected
+
     def set attribute, value
       @mutex.synchronize do
         @params || {}
@@ -259,6 +268,7 @@ module RestFtpDaemon
     def prepare
       # Update job status
       newstatus :prepare
+      @runs += 1
 
       # Init
       @source_method = :file

data/lib/rest-ftp-daemon/job_queue.rb

@@ -81,10 +81,6 @@ module RestFtpDaemon
       statuses
     end
 
-    def jobs # change for accessor
-      @jobs
-    end
-
     def jobs_count
       @jobs.length
     end
@@ -106,17 +102,18 @@ module RestFtpDaemon
       @jobs.reverse.find { |item| item.id == id }
     end
 
+
     def push job
       # Check that item responds to "priorty" method
       raise "JobQueue.push: job should respond to priority method" unless job.respond_to? :priority
       raise "JobQueue.push: job should respond to id method" unless job.respond_to? :id
 
       @mutex.synchronize do
-        # Push job into the queue
-        @queue.push job
+        # Store the job into the global jobs list, if not already inside
+        @jobs.push(job) unless @jobs.include?(job)
 
-        # Store the job into the global jobs list
-        @jobs.push job
+        # Push job into the queue, if not already inside
+        @queue.push(job) unless @queue.include?(job)
 
         # Inform the job that it's been queued
         job.set_queued if job.respond_to? :set_queued
@@ -133,8 +130,9 @@ module RestFtpDaemon
         end
       end
     end
-    alias << push
-    alias enq push
+    alias <<      push
+    alias enq     push
+    alias requeue push
 
     def pop non_block=false
       @mutex.synchronize do

data/lib/rest-ftp-daemon/notification.rb

@@ -84,7 +84,7 @@ module RestFtpDaemon
 
       # Handle exceptions
       rescue StandardError => ex
-        log_error "EXCEPTION: #{ex.inspect}"
+        log_error "NOTIFICATION EXCEPTION: #{ex.inspect}"
     end
 
     def log_context

data/lib/rest-ftp-daemon/paginate.rb

@@ -49,7 +49,7 @@ module RestFtpDaemon
     def link p
       klass = (p == @page)? " btn-info" : ""
 
-      "<a class='page btn btn-default%s' href='?only=%s&page=%d'>%p</a>" % [
+      "<a class='paginate btn btn-default%s' href='?only=%s&page=%d'>%p</a>" % [
         klass,
         @only,
         p,

data/lib/rest-ftp-daemon/remote_sftp.rb

@@ -24,7 +24,13 @@ module RestFtpDaemon
       verbosity = @debug ? Logger::INFO : false
 
       # Connect remote server
-      @sftp = Net::SFTP.start(@url.host, @url.user, password: @url.password, verbose: verbosity, port: @url.port)
+      @sftp = Net::SFTP.start(@url.host.to_s, @url.user.to_s,
+          password: @url.password.to_s,
+          verbose: verbosity,
+          port: @url.port,
+          non_interactive: true,
+          #timeout: 3
+          )
     end
 
     def present? target

data/lib/rest-ftp-daemon/settings.rb

@@ -3,7 +3,7 @@ require "settingslogic"
 # Configuration class
 class Settings < Settingslogic
   # Read configuration
-  namespace defined?(APP_ENV) ? APP_ENV : "production"
+  namespace APP_ENV
   source File.exist?(APP_CONF) ? APP_CONF : Hash.new
   suppress_errors true
 
@@ -23,7 +23,7 @@ class Settings < Settingslogic
 
   def init_defaults
     # Init host if missing
-    Settings["host"] ||= `hostname`.chomp.split(".").first
+    Settings["host"] ||= `hostname`.to_s.chomp.split(".").first
 
     # Init PID file name if missing
     Settings["pidfile"] ||= "/tmp/#{APP_NICK}-#{Settings['host']}-#{Settings['port']}.pid"

data/lib/rest-ftp-daemon/static/css/main.css

@@ -21,8 +21,8 @@ body, table tr td, .fixed {
 
 .label {
   font-size: 1em;
-  padding: 3px 3px 1px 3px;
-  margin-bottom: 1px  ;
+  padding: 2px 3px 2px 3px;
+  margin-bottom: 1px;
   font-weight: normal;
   display: inline-block;
   font-family: sans-serif;
@@ -68,9 +68,6 @@ a.class {
 }
 
 
-
-
-
 /* Header styles
 -------------------------------------------------- */
 .header-indicators {
@@ -80,6 +77,12 @@ a.class {
 }
 
 
+.paginate {
+  padding-left: 7px !important;
+  padding-right: 7px !important;
+}
+
+
 /* Footer styles
 -------------------------------------------------- */
 html {

data/lib/rest-ftp-daemon/views/dashboard_jobs.haml

@@ -4,7 +4,7 @@
 
 .row
 
-  .col-md-5
+  .col-md-4
     .btn-group.btn-group-sm
       - klass = @only.empty? ? "btn-info" : ""
       %a.btn.btn-default{href: "?only=", class: klass}
@@ -15,7 +15,7 @@
         %a.btn.btn-default{href: "?only=#{status}", class: klass}
           #{status} (#{count})
 
-  .col-md-7
+  .col-md-8
     .btn-group.btn-group-sm
       = @paginate.browser
 
@@ -38,7 +38,9 @@
           %th{"min-width" => 120} error
           %th.text-right size
           %th.text-right bitrate
-          %th info
+          %th W
+          %th P
+          %th R
 
         - unless @queue.empty?
           %tbody.jobs

data/lib/rest-ftp-daemon/views/dashboard_table.haml

@@ -63,13 +63,22 @@
           = Helpers.format_bytes(bitrate, "bps")
 
     %td
-      - unless job.priority.nil?
-        .label.label-default.flag.worker-label= job.priority
-
       - unless job.wid.nil?
         .label.label-warning.flag.worker-label= job.wid
 
+    %td
+      - unless job.priority.nil?
+        .label.label-default.flag.worker-label= job.priority
+
+    %td
+      - unless job.runs.nil? || job.runs.zero?
+        .label.label-info.flag.worker-label= job.runs
 
+    -#%td
+      - unless job.priority.nil?
+        .label.label-default.flag.worker-label= job.priority
 
+      - unless job.wid.nil?
+        .label.label-warning.flag.worker-label= job.wid
 
 

data/lib/rest-ftp-daemon/worker.rb

@@ -26,7 +26,7 @@ module RestFtpDaemon
     def log_context
       {
       wid: @wid,
-      tag_1_worker_object: true
+      jid: Thread.current.thread_variable_get(:jid),
       }
     end
 
@@ -41,7 +41,7 @@ module RestFtpDaemon
       end
     end
 
-    def worker_status status, extra = ""
+    def worker_status status, job = nil
       # Update thread variables
       Thread.current.thread_variable_set :status, status
       Thread.current.thread_variable_set :updted_at, Time.now
@@ -50,7 +50,11 @@ module RestFtpDaemon
       return unless @log_worker_status_changes
 
       # Log this status change
-      log_info "worker: #{status} #{extra}"
+      if job.is_a?(Job)
+        log_info "#{status} - job[#{job.id}] status[#{job.status}] error[#{job.error}]"
+      else
+        log_info "#{status}"
+      end
     end
 
     def worker_jid jid

data/lib/rest-ftp-daemon/worker_conchita.rb

@@ -37,7 +37,7 @@ module RestFtpDaemon
       GC.start if @conchita["garbage_collector"]
 
     rescue StandardError => e
-      log_error "EXCEPTION: #{e.inspect}"
+      log_error "CONCHITA EXCEPTION: #{e.inspect}"
       sleep 1
     else
       # Restore previous status

data/lib/rest-ftp-daemon/worker_job.rb

@@ -16,13 +16,50 @@ module RestFtpDaemon
   protected
 
     def work
-      # Wait for a job to come into the queue
+      # Wait for a job to be available in the queue
       worker_status WORKER_STATUS_WAITING
-      #log_info "waiting"
       job = $queue.pop
 
-      # Prepare the job for processing
-      worker_status WORKER_STATUS_RUNNING, "job [#{job.id}]"
+      # Work on this job
+      work_on_job(job)
+
+      # Clean job status and sleep for 1s
+      job.wid = nil
+      sleep 1
+
+      # If job status requires a retry, just restack it
+      on_errors = Settings.at(:retry, :on_errors)
+      max_age = Settings.at(:retry, :max_age)
+      max_runs = Settings.at(:retry, :max_runs)
+
+      if !job.error
+        #log_info "job succeeded"
+
+      elsif !(on_errors.is_a?(Enumerable) && on_errors.include?(job.error.to_s))
+        log_error "not retrying: error not eligible"
+
+      elsif max_age && (job.age >= max_age)
+        log_error "not retrying: too old (max_age: #{max_age})"
+
+      elsif max_runs && (job.runs >= max_runs)
+        log_error "not retrying: too many runs (max_runs: #{max_runs})"
+
+      else
+        log_info "retrying job: requeued"
+        $queue.requeue job
+      end
+
+      # Clean worker status
+      worker_jid nil
+
+    rescue StandardError => ex
+      log_error "WORKER UNHANDLED EXCEPTION: #{ex.message}", ex.backtrace
+      worker_status WORKER_STATUS_CRASHED
+    end
+
+    def work_on_job job
+      # Prepare job and worker for processing
+      worker_status WORKER_STATUS_RUNNING, job
       worker_jid job.id
       job.wid = Thread.current.thread_variable_get :wid
 
@@ -32,9 +69,7 @@ module RestFtpDaemon
       end
 
       # Processing done
-      worker_status WORKER_STATUS_FINISHED, "job [#{job.id}]"
-      worker_jid nil
-      job.wid = nil
+      worker_status WORKER_STATUS_FINISHED, job
 
       # Increment total processed jobs count
       $queue.counter_inc :jobs_processed
@@ -42,24 +77,18 @@ module RestFtpDaemon
     rescue RestFtpDaemon::JobTimeout => ex
       log_error "JOB TIMED OUT", ex.backtrace
       worker_status WORKER_STATUS_TIMEOUT
-      worker_jid nil
-      job.wid = nil
 
+      # Inform the job
       job.oops_you_stop_now ex unless job.nil?
-      sleep 1
 
     rescue StandardError => ex
-      log_error "JOB UNHDNALED EXCEPTION: #{ex.message}", ex.backtrace
+      log_error "JOB UNHANDLED EXCEPTION: #{ex.message}", ex.backtrace
       worker_status WORKER_STATUS_CRASHED
-      job.oops_after_crash ex unless job.nil?
-      sleep 1
 
-    else
-      # Clean job status
-      job.wid = nil
+      # Inform the job
+      job.oops_after_crash ex unless job.nil?
     end
 
-
     if Settings.newrelic_enabled?
       add_transaction_tracer :work,       category: :task
     end

data/lib/rest-ftp-daemon/worker_pool.rb

@@ -71,7 +71,7 @@ module RestFtpDaemon
       @conchita = create_conchita_thread
 
     rescue StandardError => ex
-      log_error "UNHDNALED EXCEPTION: #{ex.message}", ex.backtrace
+      log_error "UNHANDLED EXCEPTION: #{ex.message}", ex.backtrace
     end
 
     def create_worker_thread wid
@@ -80,7 +80,7 @@ module RestFtpDaemon
           worker = JobWorker.new wid
           log_info "JobWorker [#{wid}]: #{worker}"
         rescue StandardError => ex
-          log_error "EXCEPTION: #{ex.message}"
+          log_error "JobWorker EXCEPTION: #{ex.message}"
         end
       end
     end
@@ -91,13 +91,11 @@ module RestFtpDaemon
           worker = ConchitaWorker.new :conchita
           log_info "ConchitaWorker: #{worker}"
         rescue StandardError => ex
-          log_error "EXCEPTION: #{ex.message}"
+          log_error "ConchitaWorker EXCEPTION: #{ex.message}"
         end
       end
     end
 
-
-
   if Settings.newrelic_enabled?
     add_transaction_tracer :create_conchita_thread,     category: :task
     add_transaction_tracer :create_worker_thread,       category: :task

data/rest-ftp-daemon.gemspec

@@ -39,6 +39,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "settingslogic"
   spec.add_runtime_dependency "haml"
   spec.add_runtime_dependency "json"
+  spec.add_runtime_dependency "net-ssh", "~> 2.10.1.rc1"
   spec.add_runtime_dependency "net-sftp"
   spec.add_runtime_dependency "double-bag-ftps"
   spec.add_runtime_dependency "facter"

data/rest-ftp-daemon.yml.sample

@@ -7,11 +7,16 @@ defaults: &defaults
   #host: "myhost"
 
   transfer:
-    notify_after_sec: 5       # wait at least X seconds between HTTP notifications
-    mkdir: true               # build directory tree if missing
-    tempfile: true            # transfer to temporary file, rename after sucessful transfer
-    overwrite: false          # overwrite any target file with the same name
-    timeout: 1800             # jobs running for longer than X seconds will be killed
+    # notify_after_sec: 5       # wait at least X seconds between HTTP notifications
+    # mkdir: true               # build directory tree if missing
+    # tempfile: true            # transfer to temporary file, rename after sucessful transfer
+    # overwrite: false          # overwrite any target file with the same name
+    # timeout: 1800             # jobs running for longer than X seconds will be killed
+
+  retry:
+    # on_errors: ["net_temp_error", "conn_reset_by_peer", "conn_timed_out", "conn_refused"]
+    # max_runs: 5
+    # max_age: 1800
 
   conchita:
     # timer: 60
@@ -26,9 +31,10 @@ defaults: &defaults
     app_name: "rftpd-bigbusiness-dev"     # nickname used for naming app
 
   debug:
-    ftp: false
-    sftp: false
-    conchita: false
+    # ftp: false
+    # sftp: false
+    # conchita: false
+    # allow_reload: false
 
   logs:
     thin:    "/var/log/rftpd-environment-thin.log"

data/spec/spec_helper.rb

@@ -57,10 +57,16 @@ RSpec.configure do |config|
 
   config.before :suite do
     call_server(:start)
-    sleep 2
+    puts
+    puts ">> waiting for server to bootup"
+    puts
+    sleep 5
   end
 
   config.after :suite do
+    puts
+    puts ">> shutting sown server"
+    puts
     call_server(:stop)
   end
 end

data/spec/support/config.yml

@@ -3,7 +3,7 @@ test:
   port: 5678
   workers: 2
   adminpwd: "admin"
-  host: 127.0.0.1
+  host: local
 
   transfer:
     notify_after_sec: 5
@@ -16,10 +16,10 @@ test:
     ftp: false
 
   logs:
-    thin:     "./tmp/log/rftpd-environment-thin.log"
-    queue:    "./tmp/log/rftpd-environment-core.log"
-    api:      "./tmp/log/rftpd-environment-core.log"
-    workers:  "./tmp/log/rftpd-environment-work.log"
-    jobs:     "./tmp/log/rftpd-environment-work.log"
-    notify:   "./tmp/log/rftpd-environment-work.log"
-    newrelic: "/var/log/rftpd-environment-newrelic.log"
+    thin:     "/tmp/rftpd-tests-thin.log"
+    queue:    "/tmp/rftpd-tests-core.log"
+    api:      "/tmp/rftpd-tests-core.log"
+    workers:  "/tmp/rftpd-tests-work.log"
+    jobs:     "/tmp/rftpd-tests-work.log"
+    notify:   "/tmp/rftpd-tests-work.log"
+    newrelic: "/tmp/rftpd-tests-newrelic.log"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: 0.231.1
+  version: 0.240.0
 platform: ruby
 authors:
 - Bruno MEDICI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-26 00:00:00.000000000 Z
+date: 2015-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -178,6 +178,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: net-ssh
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.10.1.rc1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.10.1.rc1
 - !ruby/object:Gem::Dependency
   name: net-sftp
   requirement: !ruby/object:Gem::Requirement