rest-ftp-daemon 0.202.2 → 0.210.0

data/lib/rest-ftp-daemon/views/dashboard_table.haml

@@ -9,18 +9,12 @@
   - errmsg = job.get :error_message
   - method = job.get(:target_method)
 
-  - if !job.error.nil?
-    - trclass = "danger"
-  - elsif job.status == JOB_STATUS_UPLOADING
-    - trclass = "info"
-  - elsif job.status == JOB_STATUS_FINISHED
-    - trclass = "success"
-  - elsif job.status == JOB_STATUS_QUEUED
-    - trclass = "active"
-  - else
+  - trclass = JOB_STYLES[job.status]
+
+  - unless job.error.nil?
     - trclass = "warning"
 
-  %tr{class: trclass}
+  %tr{class: trclass.to_s}
 
     %td{title: presented.to_json}
       %b= job.id
@@ -78,7 +72,7 @@
 
     %td
       - unless job.priority.nil?
-        .label.label-info.flag.worker-label= job.priority
+        .label.label-default.flag.worker-label= job.priority
 
       - unless job.wid.nil?
         .label.label-warning.flag.worker-label= job.wid

data/lib/rest-ftp-daemon/views/dashboard_workers.haml

@@ -1,5 +1,3 @@
-- styles = {waiting: :success, processing: :info, crashed: :error, done: :success }
-
 %h2 Workers
 
 %table.table.table-striped.table-hover.table-condensed
@@ -8,22 +6,26 @@
     %th ID
     %th status
     %th job
-    %th.text-right updated
+    %th.text-right seen
+
+  - @worker_variables.each do |vars|
+    - wid = vars[:wid]
+    - status = vars[:status]
+    - alive = $pool.worker_alive? wid
+    - trclass = WORKER_STYLES[status]
 
-  - @worker_vars.each do |wid, vars|
-    -# status = vars[:status]
-    - style = styles[status] || ""
+    - unless alive
+      - trclass = "danger"
+      - status = "DEAD"
 
     - if vars[:updted_at].is_a? Time
       - no_news_for = (Time.now - vars[:updted_at]).round(0)
     - else
       - no_news_for = "?"
 
-    %tr{class: style.to_s}
+    %tr{class: trclass.to_s}
       %td= wid
-      %td= vars[:status]
-      %td= vars[:jobid]
+      %td= status
+      %td= vars[:jid]
       %td.text-right
-        = no_news_for
-        s
-
+        = "#{no_news_for} s"

data/lib/rest-ftp-daemon/worker_pool.rb

@@ -1,5 +1,3 @@
-
-
 module RestFtpDaemon
   class WorkerPool
 
@@ -9,33 +7,37 @@ module RestFtpDaemon
       # Logger
       @logger = RestFtpDaemon::LoggerPool.instance.get :workers
 
-
       # Check parameters
-      raise "A thread count of #{number_threads} is less than one" if number_threads < 1
+      raise "at least one worker is needed to continue (#{number_threads} is less than one)" if number_threads < 1
 
       # Prepare status hash and vars
       @statuses = {}
       @workers = {}
       @mutex = Mutex.new
       @counter = 0
+      @timeout = (Settings.transfer.timeout rescue nil) || DEFAULT_WORKER_TIMEOUT
 
       # Create worker threads
-      info "WorkerPool initializing with #{number_threads} workers"
+      info "WorkerPool initializing with [#{number_threads}] workers and [#{@timeout}]s timeout"
       create_worker_threads number_threads
 
     end
 
-    def worker_vars
-      vars = {}
-
-      @workers.each do |name, thread|
-        #currents[thread.id] = thread.current
-        vars[thread[:name]] = thread[:vars]
+    def worker_variables
+      @workers.collect do |wid, worker|
+        vars = {}
+        worker.thread_variables.each do |var|
+          vars[var] = worker.thread_variable_get var
+        end
+        vars
       end
+    end
 
-      vars
+    def worker_alive? wid
+      @workers[wid] && @workers[wid].alive?
     end
 
+
   private
 
     def create_worker_threads n
@@ -46,104 +48,92 @@ module RestFtpDaemon
         end
 
         # Create a dedicated thread for this worker
-        name = "w#{@counter}"
-        @workers[name] = create_worker_thread name
+        wid = "w#{@counter}"
+        @workers[wid] = create_worker_thread wid
       end
-
     end
-    def create_worker_thread name
-              @workers[name] = Thread.new name do
+
+    def create_worker_thread wid
+      Thread.new wid do
 
         # Set thread context
-        Thread.current[:name] = name
-        Thread.current[:vars] = { started_at: Time.now }
+        Thread.current.thread_variable_set :wid, wid
+        Thread.current.thread_variable_set :started_at, Time.now
 
         # Start working
-        work
-      end
+        worker_status :starting
+        loop do
+          work
+        end
 
+      end
     end
 
     def work
-      worker_status :starting
-
-      loop do
-
-        begin
-          info "waiting for a job"
-
-          # Wait for a job to come into the queue
-          worker_status :waiting
-          job = $queue.pop
-
-          # Do the job
-          info "processing [#{job.id}]"
-
-          worker_status :processing, job.id
-          job.wid = Thread.current[:name]
-          job.process
-          info "processed [#{job.id}]"
-          job.wid = nil
-          worker_status :done
-
-          # Increment total processed jobs count
-          $queue.counter_inc :jobs_processed
-
-        rescue Exception => ex
-          handle_job_uncaught_exception job, ex
-
-        else
-          # Clean job status
-          worker_status :free
-          job.wid = nil
-
-        end
-
+      # Wait for a job to come into the queue
+      worker_status :waiting
+      info "waiting for a job"
+      job = $queue.pop
+
+      # Prepare the job for processing
+      worker_status :processing
+      info "job processing"
+      worker_jid job.id
+      job.wid = Thread.current.thread_variable_get :wid
+
+      # Processs this job protected by a timeout
+      status = Timeout::timeout(@timeout, RestFtpDaemon::JobTimeout) do
+        job.process
       end
-    end
 
-    def handle_job_uncaught_exception job, ex
-      begin
-        # Log the exception
-        info "UNHDNALED EXCEPTION: job: #{ex.message}", lines: ex.backtrace
+      # Processing done
+      worker_status :done
+      info "job processed"
+      worker_jid nil
+      job.wid = nil
 
-        # Tell the worker has creashed
-        worker_status :crashed
+      # Increment total processed jobs count
+      $queue.counter_inc :jobs_processed
 
-        # Flag the job as crashed
-        job.oops_after_crash ex
+    rescue RestFtpDaemon::JobTimeout => ex
+      info "JOB TIMED OUT", lines: ex.backtrace
+      worker_status :timeout
+      job.oops_you_stop_now ex unless job.nil?
+      sleep 1
 
-      rescue Exception => ex
-        info "DOUBLE EXCEPTION: #{ex.message}", lines: ex.backtrace
+    rescue Exception => ex
+      info "[#{job.id}] UNHDNALED EXCEPTION: #{ex.message}", lines: ex.backtrace
+      worker_status :crashed
+      job.oops_after_crash ex unless job.nil?
+      sleep 1
 
-      end
+    else
+      # Clean job status
+      worker_status :free
+      job.wid = nil
 
-      # Wait a bit
-      sleep 1
     end
 
-
   protected
 
-    def ping
-    end
-
     def info message, context = {}
       return if @logger.nil?
 
-      # Ensure context is a hash of options and inject context
-      context = {} unless context.is_a? Hash
-      context[:id] = Thread.current[:name]
-      context[:origin] = self.class
-
       # Forward to logger
-      @logger.info_with_id message, context
+      @logger.info_with_id message,
+        wid: Thread.current.thread_variable_get(:wid),
+        jid: Thread.current.thread_variable_get(:jid),
+        origin: self.class.to_s
+    end
+
+    def worker_status status
+      Thread.current.thread_variable_set :status, status
+      Thread.current.thread_variable_set :updted_at, Time.now
     end
 
-    def worker_status status, jobid = nil
-      Thread.current[:vars][:status] = status
-      Thread.current[:vars][:jobid] = jobid
-      Thread.current[:vars][:updted_at] = Time.now
+    def worker_jid jid
+      Thread.current.thread_variable_set :jid, jid
+      Thread.current.thread_variable_set :updted_at, Time.now
     end
 
   end

data/rest-ftp-daemon.yml.sample

@@ -9,17 +9,18 @@ defaults: &defaults
 
   transfer:
     update_every_kb: 1024
-    notify_after_sec: 10
-
+    notify_after_sec: 5
     mkdir: true
     tempfile: true
     overwrite: false
+    timeout: 1800
 
 
   conchita:
     timer: 10
     #clean_failed: 3600
-    #clean_finished: 600
+    #clean_finished: 3600
+    #clean_queued: 86400
 
   debug:
     ftp: false
@@ -28,7 +29,8 @@ defaults: &defaults
     thin:    "/var/log/rftpd-environment-thin.log"
     queue:   "/var/log/rftpd-environment-core.log"
     api:     "/var/log/rftpd-environment-core.log"
-    workers: "/var/log/rftpd-environment-work.log"
+    workers: "/var/log/rftpd-environment-core.log"
+    jobs:    "/var/log/rftpd-environment-work.log"
     notify:  "/var/log/rftpd-environment-work.log"
 
 preprod:

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: 0.202.2
+  version: 0.210.0
 platform: ruby
 authors:
 - Bruno MEDICI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-19 00:00:00.000000000 Z
+date: 2015-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler