rest-ftp-daemon 0.202.2 → 0.210.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a84dfb4963cc92d2a504f0fbad4670ea68f37fce
-  data.tar.gz: f6dfe45e4adb40227e70396df77b2b3691a78fd6
+  metadata.gz: ad1215dadf62110ff9e20ee3e5100c7af016c258
+  data.tar.gz: e5e14d4cd2f4bf5170915e4a8ab0f2fde3046c59
 SHA512:
-  metadata.gz: b8d407b7aaef242722cd7e60bea9829650fcb55f21660471ac2335dd24383d5100f30f15e75e6cfe995718f166e3efc3c01e5f7b2f40e3147fdc8e529dc48602
-  data.tar.gz: a5f1c2d24a5498017c5c6c6eaa34779950006648e0d8e57b59bd8f59836bd41168538c34a3e39103ed887fc4fe36b6e591bc679c605740956124a8ac504bbac5
+  metadata.gz: 0f67f2a19ef8e17abc200bce126192127595c8c8e3c712840e23fffbf462bd0b06221f1e3c828c4fe46c334f834bc8312de60b42d9cdedd02f25ac63d0028dd2
+  data.tar.gz: af5f5a20d55f85cbe5213f2db1f6ed5593c538283c2add01608a34eff6a20e322620f0123b365457859f9d731bf6a6a0efdb6c2a08556ac4bf39fe64ce921a89

data/.gitignore

@@ -3,5 +3,5 @@ pkg
 .DS_Store
 *.log
 tmp/
-work/
-rest-ftp-daemon.yml
+DOC/
+rest-ftp-daemon.yml

data/lib/rest-ftp-daemon.rb

@@ -5,17 +5,24 @@ require 'grape'
 require 'grape-entity'
 require 'haml'
 require 'facter'
+require 'uri'
+require 'securerandom'
+require 'timeout'
 require 'sys/cpu'
 require 'syslog'
+require 'net/ftp'
+require 'net/http'
+require 'double_bag_ftps'
 
 
-# My libs
+# Project's libs
 require 'rest-ftp-daemon/constants'
 require 'rest-ftp-daemon/settings'
 require 'rest-ftp-daemon/exceptions'
 require 'rest-ftp-daemon/helpers'
 require 'rest-ftp-daemon/uri'
 require 'rest-ftp-daemon/job_queue'
+require 'rest-ftp-daemon/worker'
 require 'rest-ftp-daemon/worker_pool'
 require 'rest-ftp-daemon/logger'
 require 'rest-ftp-daemon/logger_pool'

data/lib/rest-ftp-daemon/api/dashboard.rb

@@ -3,6 +3,12 @@ module RestFtpDaemon
     class Root < Grape::API
 
 
+####### HELPERS
+
+      helpers do
+      end
+
+
 ####### DASHBOARD - GET /
 
       # Server global status
@@ -15,14 +21,18 @@ module RestFtpDaemon
         # Detect QS filters
         only = params["only"].to_s
 
-        # Get jobs to display
-        jobs = $queue.sorted_by_status(only)
+        # Get jobs for this view, order jobs by their weights
+        current = $queue.filter_jobs only
+
+        # Provide queue only if no filtering set
+        queue = []
+        queue = $queue.queue if only.empty?
 
         # Get workers status
-        @worker_vars = $pool.worker_vars
+        @worker_variables = $pool.worker_variables
 
         # Compile haml template
-        output = render :dashboard, {jobs: jobs, only: only}
+        output = render :dashboard, {queue: queue, current: current, only: only}
 
         # Send response
         env['api.format'] = :html

data/lib/rest-ftp-daemon/api/jobs.rb

@@ -13,7 +13,8 @@ module RestFtpDaemon
 
         begin
           # Get job to display
-          job = job_find params[:id]
+          raise RestFtpDaemon::JobNotFound if params[:id].nil?
+          job = $queue.find_by_id(params[:id]) || $queue.find_by_id(params[:id], true)
           raise RestFtpDaemon::JobNotFound if job.nil?
 
         rescue RestFtpDaemon::JobNotFound => exception
@@ -48,7 +49,8 @@ module RestFtpDaemon
           only = params["only"].to_s
 
           # Get jobs to display
-          jobs = $queue.sorted_by_status(only)
+          # jobs = $queue.sorted_by_status(only)
+          jobs = $queue.jobs
 
         rescue RestFtpDaemonException => exception
           info "EXCEPTION: RestFtpDaemonException: #{exception.message}"
@@ -86,8 +88,8 @@ module RestFtpDaemon
       end
 
       post '/jobs/' do
-        info "POST /jobs #{params.inspect}"
-        # request.body.rewind
+        info "POST /jobs", params.collect {|name, value| "#{name}: #{value.inspect}"}
+
         begin
 
           # Create a new job

data/lib/rest-ftp-daemon/api/root.rb

@@ -33,8 +33,10 @@ module RestFtpDaemon
 
       helpers do
 
-        def info message, context = {}
-          Root.logger.info_with_id message, context
+        def info message, lines = []
+          Root.logger.info_with_id message,
+            lines: lines,
+            origin: self.class.to_s
         end
 
         def api_error exception
@@ -50,13 +52,6 @@ module RestFtpDaemon
           haml_engine.render(binding, values)
         end
 
-        def job_find job_id
-          return nil if ($queue.all_size==0)
-
-          # Find a job with exactly this id, or prefixed if not found
-          $queue.find_by_id(job_id) || $queue.find_by_id(job_id, true)
-        end
-
       end
 
     end

data/lib/rest-ftp-daemon/api/status.rb

@@ -17,10 +17,9 @@ module RestFtpDaemon
           uptime: (Time.now - APP_STARTED).round(1),
           counters: $queue.counters,
           status: $queue.counts_by_status,
-          vars: $pool.worker_vars,
-          jobs_count: $queue.all_size,
-          jobs_queued: $queue.queued.collect(&:id),
-          jobs_popped: $queue.popped.collect(&:id),
+          workers: $pool.worker_variables,
+          jobs_count: $queue.jobs_count,
+          jobs_queued: $queue.queued_ids
           #routes: RestFtpDaemon::API::Root::routes,
           }
       end

data/lib/rest-ftp-daemon/constants.rb

@@ -1,31 +1,53 @@
 # Terrific constants
 APP_NAME = "rest-ftp-daemon"
-APP_VER = "0.202.2"
+APP_VER = "0.210.0"
 
 
-# Logging
-DEFAULT_LOGS_PIPE_LEN = 10
-DEFAULT_LOGS_ID_LEN = 8
-DEFAULT_LOGS_TRIM_LINE = 80
+# Jobs and workers
+JOB_RANDOM_LEN          = 8
+JOB_IDENT_LEN           = 4
+JOB_TEMPFILE_LEN        = 8
+JOB_STATUS_UPLOADING    = :uploading
+JOB_STATUS_RENAMING     = :renaming
+JOB_STATUS_FINISHED     = :finished
+JOB_STATUS_FAILED       = :failed
+JOB_STATUS_QUEUED       = :queued
 
 
-# Jobs identifiers length
-JOB_RANDOM_LEN = 8
-JOB_TEMPFILE_LEN = 8
-JOB_IDENT_LEN = 4
+# Logging and startup
+LOG_PIPE_LEN            = 10
+LOG_COL_WID             = 4
+LOG_COL_JID             = JOB_IDENT_LEN+3+2
+LOG_COL_ID              = 6
+LOG_TRIM_LINE           = 80
 
 
-# Jobs
-JOB_UPDATE_KB = 2048
-JOB_STATUS_UPLOADING = :uploading
-JOB_STATUS_FINISHED = :finished
-JOB_STATUS_QUEUED = :queued
-JOB_WEIGHTS = {queued: -10, uploading: 10, finished: 50}
-
 # Notifications
-NOTIFY_PREFIX = "rftpd"
-NOTIFY_USERAGENT = "#{APP_NAME} - #{APP_VER}"
-NOTIFY_IDENTIFIER_LEN = 4
+NOTIFY_PREFIX           = "rftpd"
+NOTIFY_USERAGENT        = "#{APP_NAME} - #{APP_VER}"
+NOTIFY_IDENTIFIER_LEN   = 4
+
+
+# Dashboard row styles
+JOB_STYLES = {
+  JOB_STATUS_QUEUED     => :active,
+  JOB_STATUS_FAILED     => :warning,
+  JOB_STATUS_FINISHED   => :success,
+  JOB_STATUS_UPLOADING  => :info,
+  JOB_STATUS_RENAMING   => :info,
+  }
+WORKER_STYLES = {
+  :waiting              => :success,
+  :processing           => :info,
+  :crashed              => :danger,
+  :done                 => :success,
+  :dead                 => :danger
+  }
+
+
+# Configuration defaults
+DEFAULT_WORKER_TIMEOUT  = 3600
+DEFAULT_FTP_CHUNK       = 2048
 
 
 # Initialize defaults

data/lib/rest-ftp-daemon/exceptions.rb

@@ -8,6 +8,7 @@ module RestFtpDaemon
   class MissingPool               < RestFtpDaemonException; end
 
   class JobException              < RestFtpDaemonException; end
+  class JobTimeout                < RestFtpDaemonException; end
   class JobNotFound               < RestFtpDaemonException; end
   class JobUnresolvedTokens       < RestFtpDaemonException; end
   class JobAssertionFailed        < RestFtpDaemonException; end

data/lib/rest-ftp-daemon/helpers.rb

@@ -1,5 +1,3 @@
-require 'securerandom'
-
 module RestFtpDaemon
   class Helpers
 
@@ -96,15 +94,15 @@ module RestFtpDaemon
     # Dates and times: date with time generator
     def self.datetime_full datetime
       return "-"  if datetime.nil?
-      return datetime.to_datetime.strftime("%d.%m.%Y %H:%M")
+      return datetime.to_datetime.strftime("%d.%m.%Y %H:%M:%S")
     end
 
     def self.datetime_short datetime
       # return param.class
       return "-" if datetime.nil?
       return "?" unless datetime.respond_to? :to_date
-      return datetime.to_datetime.strftime("%H:%M") if datetime.to_date == Time.now.to_date
-      return datetime.to_datetime.strftime("%d/%m %H:%M")
+      return datetime.to_datetime.strftime("%H:%M:%S") if datetime.to_date == Time.now.to_date
+      return datetime.to_datetime.strftime("%d/%m %H:%M:%S")
     end
 
     def self.hide_credentials_from_url url

data/lib/rest-ftp-daemon/job.rb

@@ -1,8 +1,3 @@
-require 'uri'
-require 'net/ftp'
-require 'double_bag_ftps'
-require 'timeout'
-
 module RestFtpDaemon
   class Job
 
@@ -40,11 +35,8 @@ module RestFtpDaemon
       @status = nil
       @wid = nil
 
-      # Debug mode
-      @ftp_debug_enabled = (Settings.at :debug, :ftp) == true
-
       # Logger
-      @logger = RestFtpDaemon::LoggerPool.instance.get :workers
+      @logger = RestFtpDaemon::LoggerPool.instance.get :jobs
 
       # Protect with a mutex
       @mutex = Mutex.new
@@ -59,22 +51,29 @@ module RestFtpDaemon
       flag_default :overwrite, false
       flag_default :tempfile, false
 
+      # Read source file size and parameters
+      @ftp_debug_enabled = (Settings.at :debug, :ftp) == true
+      update_every_kb = (Settings.transfer.update_every_kb rescue nil) || DEFAULT_FTP_CHUNK
+      @notify_after_sec = Settings.transfer.notify_after_sec rescue nil
+      @chunk_size = update_every_kb * 1024
+
       # Flag current job
       @queued_at = Time.now
+      @updated_at = Time.now
 
       # Send first notification
-      info "Job.initialize notify: queued"
+      info "Job.initialize notify[queued] notify_after_sec[#{@notify_after_sec}] update_every_kb[#{@update_every_kb}]"
       client_notify :queued
     end
 
     def process
       # Update job's status
       @error = nil
+      info "Job.process starting"
 
       # Prepare job
       begin
-        info "Job.process prepare"
-        newstatus :preparing
+        newstatus :prepare
         prepare
 
       rescue RestFtpDaemon::JobMissingAttribute => exception
@@ -95,22 +94,25 @@ module RestFtpDaemon
       rescue RestFtpDaemon::JobAssertionFailed => exception
         return oops :started, exception, :assertion_failed
 
-      rescue RestFtpDaemon::RestFtpDaemonException => exception
-        return oops :started, exception, :prepare_failed, true
+      # rescue RestFtpDaemon::JobTimeout => exception
+      #   info "Job.process propagate JobTimeout to Worker"
+      #   raise RestFtpDaemon::JobTimeout
 
-      rescue Exception => exception
-        return oops :started, exception, :prepare_unhandled, true
+      # rescue RestFtpDaemon::RestFtpDaemonException => exception
+      #   return oops :started, exception, :prepare_failed, true
+
+      # rescue Exception => exception
+      #   return oops :started, exception, :prepare_unhandled, true
 
       else
         # Prepare done !
         newstatus :prepared
-        info "Job.process notify: started"
+        info "Job.process notify[started]"
         client_notify :started
       end
 
       # Process job
       begin
-        info "Job.process transfer"
         newstatus :starting
         transfer
 
@@ -162,16 +164,20 @@ module RestFtpDaemon
       rescue RestFtpDaemon::JobAssertionFailed => exception
         return oops :ended, exception, :assertion_failed
 
-      rescue RestFtpDaemon::RestFtpDaemonException => exception
-        return oops :ended, exception, :transfer_failed, true
+      # rescue RestFtpDaemon::JobTimeout => exception
+      #   info "Job.process propagate JobTimeout to Worker"
+      #   raise RestFtpDaemon::JobTimeout
+
+      # rescue RestFtpDaemon::RestFtpDaemonException => exception
+      #   return oops :ended, exception, :transfer_failed, true
 
-      rescue Exception => exception
-        return oops :ended, exception, :transfer_unhandled, true
+      # rescue Exception => exception
+      #   return oops :ended, exception, :transfer_unhandled, true
 
       else
         # All done !
         newstatus JOB_STATUS_FINISHED
-        info "Job.process notify: ended"
+        info "Job.process notify[ended]"
         client_notify :ended
       end
 
@@ -184,16 +190,22 @@ module RestFtpDaemon
       end
     end
 
+    def weight
+      @weight = [@priority.to_i, -@queued_at.to_i]
+    end
+
     def set_queued
       # Update job status
       newstatus JOB_STATUS_QUEUED
     end
 
     def oops_after_crash exception
-      # info "Yes, we crash!"
-      return oops :crashed, exception, :crashed
+      return oops :ended, exception, :crashed
     end
 
+    def oops_you_stop_now exception
+      return oops :ended, exception, :timeout
+    end
 
   protected
 
@@ -338,8 +350,6 @@ module RestFtpDaemon
       # Handle each source file matched, and start a transfer
       done = 0
       source_matches.each do |filename|
-        # Increment counter
-
         # Do the transfer, only if it's a file
         ftp_transfer filename, target_name
 
@@ -355,21 +365,20 @@ module RestFtpDaemon
 
   private
 
-    def info message, context = {}
+    def info message, lines = []
       return if @logger.nil?
 
-      # Inject context
-      context[:id] = @id
-      context[:origin] = self.class
-
       # Forward to logger
-      @logger.info_with_id message, context
+      @logger.info_with_id message,
+        wid: @wid,
+        jid: @id,
+        lines: lines,
+        origin: self.class.to_s
     end
 
     def newstatus name
-      # Update local status
       @status = name
-      # push_job
+      worker_is_still_active
     end
 
     def flag_default name, default
@@ -518,7 +527,7 @@ module RestFtpDaemon
       end
 
       # Now we were able to chdir inside, just tell it
-      info "#{pref} changed to [#{@ftp.pwd}]"
+      info "#{pref} > ftp.pwd [#{@ftp.pwd}]"
     end
 
     def ftp_presence target_name
@@ -565,10 +574,6 @@ module RestFtpDaemon
         end
       end
 
-      # Read source file size and parameters
-      update_every_kb = (Settings.transfer.update_every_kb rescue nil) || JOB_UPDATE_KB
-      notify_after_sec = Settings.transfer.notify_after_sec rescue nil
-
       # Compute temp target name
       target_real = target_name
       if @tempfile
@@ -577,90 +582,103 @@ module RestFtpDaemon
       end
 
       # Start transfer
-      chunk_size = update_every_kb * 1024
-      t0 = tstart = Time.now
-      notified_at = Time.now
+      transfer_started_at = Time.now
+      @transfer_pointer_at = transfer_started_at
+
+      @notified_at = Time.now
       newstatus JOB_STATUS_UPLOADING
 
-      @ftp.putbinaryfile(source_filename, target_real, chunk_size) do |block|
-        # Update counters
-        @transfer_sent += block.bytesize
-        set :transfer_sent, @transfer_sent
-
-        # Update bitrate
-        #dt = Time.now - t0
-        bitrate0 = get_bitrate(chunk_size, t0).round(0)
-        set :transfer_bitrate, bitrate0
-
-        # Update job info
-        percent0 = (100.0 * @transfer_sent / @transfer_total).round(0)
-        set :progress, percent0
-
-        # Log progress
-        stack = []
-        stack << "#{percent0} %"
-        stack << (Helpers.format_bytes @transfer_sent, "B")
-        stack << (Helpers.format_bytes @transfer_total, "B")
-        stack << (Helpers.format_bytes bitrate0, "bps")
-        info "Job.ftp_transfer" + stack.map{|txt| ("%#{DEFAULT_LOGS_PIPE_LEN.to_i}s" % txt)}.join("\t")
-
-        # Update time pointer
-        t0 = Time.now
-
-        # Notify if requested
-        unless notify_after_sec.nil? || (notified_at + notify_after_sec > Time.now)
-          notif_status = {
-            progress: percent0,
-            transfer_sent: @transfer_sent,
-            transfer_total: @transfer_total,
-            transfer_bitrate: bitrate0
-            }
-          client_notify :progress, status: notif_status
-          notified_at = Time.now
-        end
+      @ftp.putbinaryfile(source_filename, target_real, @chunk_size) do |block|
+        # Update the worker activity marker
+        worker_is_still_active
 
+        # Update job status after this block transfer
+        ftp_transfer_block block
       end
 
       # Rename temp file to target_temp
       if @tempfile
-        newstatus :renaming
+        newstatus JOB_STATUS_RENAMING
         info "Job.ftp_transfer renaming: #{target_name}"
         @ftp.rename target_real, target_name
       end
 
       # Compute final bitrate
-      set :transfer_bitrate, get_bitrate(@transfer_total, tstart).round(0)
+      set :transfer_bitrate, get_bitrate(@transfer_total, transfer_started_at).round(0)
 
       # Done
       set :source_current, nil
       info "Job.ftp_transfer finished"
     end
 
+    def ftp_transfer_block block
+      # Update counters
+      @transfer_sent += block.bytesize
+      set :transfer_sent, @transfer_sent
+
+      # Update bitrate
+      #dt = Time.now - t0
+      bitrate0 = get_bitrate(@chunk_size, @transfer_pointer_at).round(0)
+      set :transfer_bitrate, bitrate0
+
+      # Update job info
+      percent0 = (100.0 * @transfer_sent / @transfer_total).round(0)
+      set :progress, percent0
+
+      # Log progress
+      stack = []
+      stack << "#{percent0} %"
+      stack << (Helpers.format_bytes @transfer_sent, "B")
+      stack << (Helpers.format_bytes @transfer_total, "B")
+      stack << (Helpers.format_bytes bitrate0, "bps")
+      stack2 = stack.map{ |txt| ("%#{LOG_PIPE_LEN.to_i}s" % txt)}.join("\t")
+      info "Job.ftp_transfer #{stack2}"
+
+      # Update time pointer
+      @transfer_pointer_at = Time.now
+
+      # Notify if requested
+      # info "Job.ftp_transfer next notif (#{(@notified_at+@notify_after_sec).to_f}) now #{Time.now.to_f}"
+      if @notify_after_sec.nil? || (Time.now > @notified_at + @notify_after_sec)
+        notif_status = {
+          progress: percent0,
+          transfer_sent: @transfer_sent,
+          transfer_total: @transfer_total,
+          transfer_bitrate: bitrate0
+          }
+        client_notify :progress, status: notif_status
+        @notified_at = Time.now
+      end
+    end
+
     def client_notify event, payload = {}
       # Skip if no URL given
       return unless @notify
 
       # Ok, create a notification!
-      begin
-        payload[:id] = @id
-        payload[:event] = event
-        RestFtpDaemon::Notification.new @notify, payload
-      rescue Exception => ex
-        info "Job.client_notify EXCEPTION: #{ex.inspect}"
-      end
-    end
+      payload[:id] = @id
+      payload[:event] = event
+      RestFtpDaemon::Notification.new @notify, payload
+
+    rescue Exception => ex
+      info "Job.client_notify EXCEPTION: #{ex.inspect}"
+     end
 
     def get_bitrate total, last_timestamp
       8*total.to_f / (Time.now - last_timestamp)
     end
 
+    def worker_is_still_active
+       Thread.current.thread_variable_set :updted_at, Time.now
+    end
+
     def oops event, exception, error = nil, include_backtrace = false
       # Log this error
       error = exception.class if error.nil?
 
       message = "Job.oops event[#{event.to_s}] error[#{error.to_s}] ex[#{exception.class}] #{exception.message}"
       if include_backtrace
-        info message, lines: exception.backtrace
+        info message, exception.backtrace
       else
         info message
       end
@@ -669,9 +687,9 @@ module RestFtpDaemon
       @ftp.close unless @ftp.nil? || @ftp.welcome.nil?
 
       # Update job's internal status
-      newstatus :failed
+      newstatus JOB_STATUS_FAILED
       @error = error
-      set :error_exception, exception.class
+      set :error_exception, exception.class.to_s
       set :error_message, exception.message
 
       # Build status stack