rest-ftp-daemon 0.214.0 → 0.220.0

data/lib/rest-ftp-daemon/worker_pool.rb

@@ -1,5 +1,7 @@
 module RestFtpDaemon
   class WorkerPool
+    include LoggerHelper
+    attr_reader :logger
 
     attr_reader :wid
 
@@ -7,28 +9,24 @@ module RestFtpDaemon
       include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
     end
 
-    def initialize number_threads
+    def initialize
       # Logger
       @logger = RestFtpDaemon::LoggerPool.instance.get :workers
-
-      # Check parameters
-      raise "at least one worker is needed to continue (#{number_threads} is less than one)" if number_threads < 1
+      # log_info "WorkerPool: log_info"
+      # log_error "WorkerPool: log_error"
+      # log_error "WorkerPool: log_error + lines", ['line1', 'line2']
 
       # Prepare status hash and vars
       @statuses = {}
       @workers = {}
       @conchita = nil
       @mutex = Mutex.new
-      @counter = 0
-      @timeout = (Settings.transfer.timeout rescue nil) || DEFAULT_WORKER_TIMEOUT
 
-      # Create worker threads
-      info "WorkerPool creating worker threads [#{number_threads}] timeout [#{@timeout}]s"
-      create_worker_threads number_threads
+      # Identifiers generator
+      @last_id = 0
 
-      # Create conchita thread
-      info "WorkerPool creating conchita thread"
-      create_conchita_thread
+      # Create worker threads
+      create_threads
     end
 
     def worker_variables
@@ -47,119 +45,65 @@ module RestFtpDaemon
 
   private
 
-    def create_worker_threads n
-# FIXME counter instead of upto ?
-      n.times do
-        # Increment counter
-        @mutex.synchronize do
-          @counter +=1
-        end
+    def generate_id
+      @mutex.synchronize do
+        @last_id += 1
+      end
+      "w#{@last_id}"
+    end
+
+    def create_threads
+      # Read configuration
+      number_threads = (Settings.workers || APP_WORKERS)
+
+      if number_threads < 1
+        log_error "create_threads: one worker is the minimum possible number (#{number_threads} configured)"
+        raise InvalidWorkerNumber
+      end
 
-        # Create a dedicated thread for this worker
-        wid = "w#{@counter}"
+      # Create workers
+      log_info "WorkerPool creating #{number_threads}x JobWorker, 1x ConchitaWorker"
+
+      # Start worker threads
+      number_threads.times do
+        wid = generate_id
         @workers[wid] = create_worker_thread wid
       end
+
+      # Start conchita thread
+      @conchita = create_conchita_thread
+
+    rescue Exception => ex
+      log_error "UNHDNALED EXCEPTION: #{ex.message}", ex.backtrace
+
     end
 
     def create_worker_thread wid
       Thread.new wid do
-        # Set thread context
-        Thread.current.thread_variable_set :wid, wid
-        Thread.current.thread_variable_set :started_at, Time.now
-        worker_status :starting
-
-        # Start working
-        loop do
-          begin
-            work
-          rescue Exception => ex
-            puts "WORKER UNEXPECTED CRASH: #{ex.message}", lines: ex.backtrace
-            sleep 1
-          end
+        begin
+          worker = JobWorker.new wid
+          log_info "JobWorker [#{wid}]: #{worker}"
+        rescue Exception => ex
+          log_error "EXCEPTION: #{ex.message}"
         end
-
-        # We should never get here
       end
     end
 
     def create_conchita_thread
       Thread.new do
         begin
-          @conchita = Conchita.new
-        rescue Exception => e
-          info "CONCHITA EXCEPTION: #{e.inspect}"
+          worker = ConchitaWorker.new :conchita
+          log_info "ConchitaWorker: #{worker}"
+        rescue Exception => ex
+          log_error "EXCEPTION: #{ex.message}"
         end
       end
     end
 
-    def work
-      # Wait for a job to come into the queue
-      worker_status :waiting
-      info "waiting for a job"
-      job = $queue.pop
-
-      # Prepare the job for processing
-      worker_status :working
-      worker_jid job.id
-      info "working"
-      job.wid = Thread.current.thread_variable_get :wid
-
-      # Processs this job protected by a timeout
-      Timeout::timeout(@timeout, RestFtpDaemon::JobTimeout) do
-        job.process
-      end
-
-      # Processing done
-      worker_status :finished
-      info "finished"
-      worker_jid nil
-      job.wid = nil
-
-      # Increment total processed jobs count
-      $queue.counter_inc :jobs_processed
-
-    rescue RestFtpDaemon::JobTimeout => ex
-      info "JOB TIMED OUT", lines: ex.backtrace
-      worker_status :timeout
-      job.oops_you_stop_now ex unless job.nil?
-      sleep 1
-
-    rescue Exception => ex
-      info "UNHDNALED EXCEPTION: #{ex.message}", lines: ex.backtrace
-      worker_status :crashed
-      job.oops_after_crash ex unless job.nil?
-      sleep 1
-
-    else
-      # Clean job status
-      worker_status :free
-      job.wid = nil
-
-    end
-
   protected
 
-    def info message, context = {}
-      return if @logger.nil?
-
-      # Forward to logger
-      @logger.info_with_id message,
-        wid: Thread.current.thread_variable_get(:wid),
-        jid: Thread.current.thread_variable_get(:jid),
-        origin: self.class.to_s
-    end
-
-    def worker_status status
-      Thread.current.thread_variable_set :status, status
-      Thread.current.thread_variable_set :updted_at, Time.now
-    end
-
-    def worker_jid jid
-      Thread.current.thread_variable_set :jid, jid
-      Thread.current.thread_variable_set :updted_at, Time.now
-    end
-
     if Settings.newrelic_enabled?
+      add_transaction_tracer :create_conchita_thread, :category => :task
       add_transaction_tracer :create_worker_thread, :category => :task
       add_transaction_tracer :work, :category => :task
     end

data/rest-ftp-daemon.yml.sample

@@ -15,10 +15,6 @@ defaults: &defaults
     overwrite: false
     timeout: 1800
 
-  # newrelic:
-    # appname: rftpd-testing
-    # license: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
-
   conchita:
     timer: 60
     # garbage_collector: true
@@ -28,6 +24,7 @@ defaults: &defaults
 
   debug:
     ftp: false
+    # license: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 
   logs:
     thin:    "/var/log/rftpd-environment-thin.log"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: 0.214.0
+  version: 0.220.0
 platform: ruby
 authors:
 - Bruno MEDICI
@@ -229,15 +229,14 @@ files:
 - lib/rest-ftp-daemon/api/job_presenter.rb
 - lib/rest-ftp-daemon/api/jobs.rb
 - lib/rest-ftp-daemon/api/root.rb
-- lib/rest-ftp-daemon/api/routes.rb
 - lib/rest-ftp-daemon/api/status.rb
-- lib/rest-ftp-daemon/conchita.rb
 - lib/rest-ftp-daemon/constants.rb
 - lib/rest-ftp-daemon/exceptions.rb
 - lib/rest-ftp-daemon/helpers.rb
 - lib/rest-ftp-daemon/job.rb
 - lib/rest-ftp-daemon/job_queue.rb
 - lib/rest-ftp-daemon/logger.rb
+- lib/rest-ftp-daemon/logger_helper.rb
 - lib/rest-ftp-daemon/logger_pool.rb
 - lib/rest-ftp-daemon/notification.rb
 - lib/rest-ftp-daemon/paginate.rb
@@ -252,6 +251,9 @@ files:
 - lib/rest-ftp-daemon/views/dashboard_table.haml
 - lib/rest-ftp-daemon/views/dashboard_tokens.haml
 - lib/rest-ftp-daemon/views/dashboard_workers.haml
+- lib/rest-ftp-daemon/worker.rb
+- lib/rest-ftp-daemon/worker_conchita.rb
+- lib/rest-ftp-daemon/worker_job.rb
 - lib/rest-ftp-daemon/worker_pool.rb
 - rest-ftp-daemon.gemspec
 - rest-ftp-daemon.yml.sample

data/lib/rest-ftp-daemon/api/routes.rb

@@ -1,16 +0,0 @@
-module RestFtpDaemon
-  module API
-    class Root < Grape::API
-
-
-####### GET /routes
-
-      get '/routes' do
-        info "GET /routes"
-        status 200
-        return RestFtpDaemon::API::Root::routes
-      end
-
-    end
-  end
-end

data/lib/rest-ftp-daemon/conchita.rb

@@ -1,65 +0,0 @@
-module RestFtpDaemon
-  class Conchita
-
-    def initialize
-      # Logger
-      @logger = RestFtpDaemon::LoggerPool.instance.get :workers
-
-      # Conchita configuration
-      @conchita = Settings.conchita
-      if @conchita.nil?
-        return info "conchita: missing conchita.* configuration"
-      elsif @conchita[:timer].nil?
-        return info "conchita: missing conchita.timer value"
-      end
-
-      # Start main loop
-      info "initialized #{@conchita.inspect}"
-      cleanup
-    end
-
-  protected
-
-    def maxage status
-      @conchita["clean_#{status.to_s}"] || 0
-    end
-
-    def cleanup
-      loop do
-        #info "cleanup"
-        # info "conchita_loop: cleanup "
-        $queue.expire JOB_STATUS_FINISHED,  maxage(JOB_STATUS_FINISHED)
-        $queue.expire JOB_STATUS_FAILED,    maxage(JOB_STATUS_FAILED)
-        $queue.expire JOB_STATUS_QUEUED,    maxage(JOB_STATUS_QUEUED)
-
-        # Force garbage collector
-        GC.start if @conchita["garbage_collector"]
-
-        # Sleep for a few seconds
-        sleep @conchita[:timer]
-      end
-    end
-
-    # def conchita_gc
-    #   # Read config state
-    #   proceed = @conchita["clean_garbage"] || false
-    #   #info "conchita_clean status[#{status.to_s}] \t maxage[#{maxage}] s"
-    #   return unless proceed
-
-    #   # Trig Ruby's garbage collector
-    #   info "conchita_gc forced garbage collecting"
-    #   GC.start
-    # end
-
-    def info message, lines = []
-      return if @logger.nil?
-
-      # Forward to logger
-      @logger.info_with_id message,
-        wid: :conchita,
-        lines: lines,
-        origin: self.class.to_s
-    end
-
-  end
-end