rest-ftp-daemon 0.250.5 → 0.300.1

data/lib/rest-ftp-daemon/views/dashboard.haml

@@ -6,7 +6,7 @@
     %link{ href:"/css/bootstrap.css" , rel: "stylesheet"}
     %link{ href:"/css/main.css" , rel: "stylesheet"}
     -# %link{ href:"http://fonts.googleapis.com/css?family=Inconsolata:400,700" , rel: "stylesheet"}
-    %title="#{Settings.host} [#{Settings.namespace}] #{APP_NAME}"
+    %title="#{Conf.host} [#{Conf.app_env}] #{Conf.app_name}"
 
   %body
 
@@ -21,7 +21,7 @@
 
       .row
         #box-tokens.col-md-6
-          = render :dashboard_tokens, {tokens: Settings.endpoints || {}}
+          = render :dashboard_tokens, {tokens: Conf[:endpoints] || {}}
 
         #box-workers.col-md-3
           = render :dashboard_workers

data/lib/rest-ftp-daemon/views/dashboard_footer.haml

@@ -17,7 +17,7 @@
 
   .btn-group.btn-group-sm
     .btn.btn-default.btn-info Host
-    .btn.btn-default= Settings.host
+    .btn.btn-default= Conf.host
 
   .btn-group.btn-group-sm
     .btn.btn-default.btn-info IP
@@ -29,4 +29,4 @@
 
   .btn-group.btn-group-sm
     .btn.btn-default.btn-info Started
-    .btn.btn-default= Helpers.datetime_short(APP_STARTED)
+    .btn.btn-default= Helpers.datetime_short(Conf.app_started)

data/lib/rest-ftp-daemon/views/dashboard_header.haml

@@ -10,9 +10,9 @@
 .navbar-header
 
   %h1
-    = APP_NAME
+    = Conf.app_name
     %small
-      = "v#{APP_VER} [#{Settings.namespace}]"
+      = "v#{Conf.app_ver} [#{Conf.app_env}]"
 
 
 .header-indicators.navbar-header.pull-right

data/lib/rest-ftp-daemon/views/dashboard_workers.haml

@@ -14,8 +14,8 @@
       %th.text-right seen
 
   %tbody
-    - variables.each do |vars|
-      - wid = vars[:wid]
+    - variables.each do |wid, vars|
+      -# wid = vars[:wid]
       - status = vars[:status]
       - alive = $pool.worker_alive? wid
       - trclass = WORKER_STYLES[status]

data/lib/rest-ftp-daemon/worker.rb

@@ -1,12 +1,8 @@
 module RestFtpDaemon
   class Worker
-    include LoggerHelper
+    include Shared::LoggerHelper
     attr_reader :logger
 
-    if Settings.newrelic_enabled?
-      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
-    end
-
     def initialize wid, pool = nil
       # Logger
       @logger = RestFtpDaemon::LoggerPool.instance.get :workers
@@ -20,15 +16,28 @@ module RestFtpDaemon
       Thread.current.thread_variable_set :wid, wid
       Thread.current.thread_variable_set :started_at, Time.now
       worker_status WORKER_STATUS_STARTING
+
+      # Load corker conf
+      load_config wid
     end
 
   protected
 
-    def log_context
-      {
-        wid: Thread.current.thread_variable_get(:wid),
-        jid: Thread.current.thread_variable_get(:jid),
-      }
+    def wait_according_to_config
+      # Sleep for a few seconds
+      worker_status WORKER_STATUS_WAITING
+      sleep @config[:timer] if @config.is_a? Hash
+    end
+
+    def log_prefix
+     [
+      Thread.current.thread_variable_get(:wid),
+      Thread.current.thread_variable_get(:jid),
+      nil
+      ]
+    end
+
+    def work
     end
 
     def start
@@ -52,9 +61,9 @@ module RestFtpDaemon
 
       # Log this status change
       if job.is_a?(Job)
-        log_info "#{status} - job[#{job.id}] status[#{job.status}] error[#{job.error}]"
+        log_info "status [#{status}] on job[#{job.id}] status[#{job.status}] error[#{job.error}]"
       else
-        log_info "#{status}"
+        log_info "status [#{status}]"
       end
     end
 
@@ -63,5 +72,27 @@ module RestFtpDaemon
       Thread.current.thread_variable_set :updated_at, Time.now
     end
 
+  private
+
+    def load_config wid
+      # My debug
+      @debug = (Conf.at :debug, wid) == true
+      @log_worker_status_changes = @debug
+
+      # My configuration
+      @config = Conf[wid]
+      if !@config.is_a? Hash
+        return log_info "#{self.class.name}: missing #{wid}/* configuration"
+      elsif @config[:timer].nil?
+        return log_info "#{self.class.name}: missing #{wid}/timer value"
+      end
+    end
+
+    # NewRelic instrumentation
+    if Conf.newrelic_enabled?
+      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+      add_transaction_tracer :work,       category: :task
+    end
+
   end
 end

data/lib/rest-ftp-daemon/worker_conchita.rb

@@ -3,29 +3,25 @@ module RestFtpDaemon
   # Worker used to clean up the queue deleting expired jobs
   class ConchitaWorker < Worker
 
-    def initialize wid = :conchita
-      # Generic worker initialize
+    def initialize wid, pool = nil
+      # Call dady and load my conf
       super
 
-      # Use debug ?
-      @debug = (Settings.at :debug, :conchita) == true
-      @log_worker_status_changes = @debug
-
-      # Conchita configuration
-      @conchita = Settings.conchita
-      if !@conchita.is_a? Hash
-        return log_info "ConchitaWorker: missing conchita.* configuration"
-      elsif @conchita[:timer].nil?
-        return log_info "ConchitaWorker: missing conchita.timer value"
-      end
-
       # Start main loop
-      log_info "ConchitaWorker starting", @conchita
+      log_info "#{self.class.name} starting", @config
       start
     end
 
   protected
 
+    # def log_prefix
+    #  [
+    #   Thread.current.thread_variable_get(:wid),
+    #   nil,
+    #   nil
+    #   ]
+    # end
+
     def work
       # Announce we are working
       worker_status WORKER_STATUS_CLEANING
@@ -36,26 +32,17 @@ module RestFtpDaemon
       $queue.expire JOB_STATUS_QUEUED,    maxage(JOB_STATUS_QUEUED),    @debug
 
       # Force garbage collector
-      GC.start if @conchita["garbage_collector"]
+      GC.start if @config["garbage_collector"]
 
     rescue StandardError => e
-      log_error "CONCHITA EXCEPTION: #{e.inspect}"
+      log_error "EXCEPTION: #{e.inspect}"
       sleep 1
     else
-      # Restore previous status
-      worker_status WORKER_STATUS_WAITING
-
-      # Sleep for a few seconds
-      sleep @conchita[:timer]
+      wait_according_to_config
     end
 
     def maxage status
-      @conchita["clean_#{status}"] || 0
-    end
-
-
-    if Settings.newrelic_enabled?
-      add_transaction_tracer :work,       category: :task
+      @config["clean_#{status}"] || 0
     end
 
   end

data/lib/rest-ftp-daemon/worker_job.rb

@@ -2,22 +2,40 @@ module RestFtpDaemon
 
   # Worker used to process Jobs
   class JobWorker < Worker
-    #attr_reader :pool
 
     def initialize wid, pool
-      # Generic worker initialize
+      # Call dady and load my conf
       super
 
-      # Timeout config
-      @timeout = (Settings.transfer.timeout rescue nil) || DEFAULT_WORKER_TIMEOUT
+      # Timeout and retry config
+      @timeout    = (Conf.at(:transfer, :timeout) rescue nil)
+      @retry      = (Conf.at(:retry) rescue {})
+
+      # Retry config
+      @retry_on_errors   = Conf.at(:retry, :on_errors)
+      @retry_max_age     = Conf.at(:retry, :max_age)
+      @retry_max_runs    = Conf.at(:retry, :max_runs)
+      @retry_delay       = Conf.at(:retry, :delay)
+
 
       # Start main loop
-      log_info "JobWorker initializing", ["wid: #{wid}", "pool: #{pool}", "timeout: #{@timeout}"]
+      log_info "JobWorker initializing", {
+        wid: wid,
+        pool: pool,
+        timeout: @timeout
+      }
       start
     end
 
   protected
 
+
+    def load_config wid
+      # Do nothing
+    end
+
+  private
+
     def work
       # Wait for a job to be available in the queue
       worker_status WORKER_STATUS_WAITING
@@ -31,26 +49,21 @@ module RestFtpDaemon
       #sleep 1
 
       # If job status requires a retry, just restack it
-      on_errors = Settings.at(:retry, :on_errors)
-      max_age = Settings.at(:retry, :max_age)
-      max_runs = Settings.at(:retry, :max_runs)
-      delay = Settings.at(:retry, :delay)
-
       if !job.error
         #log_info "job succeeded"
 
-      elsif !(on_errors.is_a?(Enumerable) && on_errors.include?(job.error))
+      elsif !(@retry_on_errors.is_a?(Enumerable) && @retry_on_errors.include?(job.error))
         log_error "not retrying: error not eligible"
 
-      elsif max_age && (job.age >= max_age)
-        log_error "not retrying: max_age reached (#{max_age} s)"
+      elsif @retry_max_age && (job.age >= @retry_max_age)
+        log_error "not retrying: max_age reached (#{@retry_max_age} s)"
 
-      elsif max_runs && (job.runs >= max_runs)
-        log_error "not retrying: max_runs reached (#{max_runs} tries)"
+      elsif @retry_max_runs && (job.runs >= @retry_max_runs)
+        log_error "not retrying: max_runs reached (#{@retry_max_runs} tries)"
 
       else
         # Delay cannot be negative, and will be 1s minimum
-        retry_after = [delay || DEFAULT_RETRY_DELAY, 1].max
+        retry_after = [@retry_delay || DEFAULT_RETRY_DELAY, 1].max
         log_info "retrying job: waiting for #{retry_after} seconds"
 
         # Wait !
@@ -71,8 +84,8 @@ module RestFtpDaemon
 
     def work_on_job job
       # Prepare job and worker for processing
-      worker_status WORKER_STATUS_RUNNING, job
       worker_jid job.id
+      worker_status WORKER_STATUS_RUNNING, job
       job.wid = Thread.current.thread_variable_get :wid
 
       # Processs this job protected by a timeout
@@ -101,9 +114,5 @@ module RestFtpDaemon
       job.oops_after_crash ex unless job.nil?
     end
 
-    if Settings.newrelic_enabled?
-      add_transaction_tracer :work,       category: :task
-    end
-
   end
 end

data/lib/rest-ftp-daemon/worker_pool.rb

@@ -2,14 +2,10 @@ module RestFtpDaemon
 
   # Handles a pool of Worker objects
   class WorkerPool
-    include LoggerHelper
+    include Shared::LoggerHelper
     attr_reader :logger
     attr_reader :wid
 
-    if Settings.newrelic_enabled?
-      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
-    end
-
     def initialize
       # Logger
       @logger = RestFtpDaemon::LoggerPool.instance.get :workers
@@ -17,93 +13,106 @@ module RestFtpDaemon
       # Prepare status hash and vars
       @statuses = {}
       @workers = {}
-      @conchita = nil
       @mutex = Mutex.new
 
       # Identifiers generator
-      @last_id = 0
+      @last_worker_id = 0
 
       # Create worker threads
       create_threads
     end
 
     def worker_variables
-      @workers.collect do |_wid, worker|
-        vars = {}
-        worker.thread_variables.each do |var|
-          vars[var] = worker.thread_variable_get var
-        end
-        vars
+      vars = {}
+      @workers.collect do |wid, worker|
+        vars[wid] = thread_variables worker
       end
+      vars
     end
 
     def worker_alive? wid
       @workers[wid] && @workers[wid].alive?
     end
 
+  protected
+
+    def log_prefix
+     [nil, nil, nil]
+    end
+
   private
 
-    def generate_id
+    def thread_variables thread
+      vars = {}
+      thread.thread_variables.each do |var|
+        vars[var] = thread.thread_variable_get var
+      end
+      vars
+    end
+
+    def next_worker_id
       @mutex.synchronize do
-        @last_id += 1
+        @last_worker_id += 1
       end
-      "w#{@last_id}"
+      "w#{@last_worker_id}"
     end
 
     def create_threads
-      # Read configuration
-      pools = (Settings.pools || {})
+      # Read configuration or initialize with empty hash
+      pools = Conf[:pools]
+      pools = {} unless pools.is_a? Hash
 
       # Minimum one worker on DEFAULT_POOL
       if !(pools.is_a? Hash)
-        log_error "create_threads: one worker is the minimum possible number (#{pools.inspect}"
+        log_error "create_threads: one JobWorker is the minimum (#{pools.inspect}"
       end
-      pools[DEFAULT_POOL] ||= 1
+      log_info "WorkerPool creating workers - JobWorker #{pools.to_hash.inspect}"
 
-      # Create workers
-      log_info "WorkerPool creating JobWorker's #{pools.inspect} + ConchitaWorker"
+      # Start ConchitaWorker and ReporterWorker
+      create_thread :conchita, ConchitaWorker
+      create_thread :reporter, ReporterWorker
+
+      # Start JobWorkers threads, ensure we have at least one worker in default pool
+      pools[DEFAULT_POOL] ||= 1
       pools.each do |pool, count|
-        # Start worker threads for each pool
-        log_info "WorkerPool creating JobWorker [#{pool}] x#{count}"
         count.times do
-          wid = generate_id
-          @workers[wid] = create_worker_thread wid, pool
+          wid = next_worker_id
+          @workers[wid] = create_thread wid, JobWorker, pool
         end
       end
 
-      # Start conchita thread
-      log_info "WorkerPool creating ConchitaWorker"
-      @conchita = create_conchita_thread
-
     rescue StandardError => ex
-      log_error "UNHANDLED EXCEPTION: #{ex.message}", ex.backtrace
-    end
-
-    def create_worker_thread wid, pool
-      Thread.new wid do
-        begin
-          worker = JobWorker.new wid, pool
-          log_info "JobWorker [#{wid}][#{pool}]: #{worker}"
-        rescue StandardError => ex
-          log_error "JobWorker EXCEPTION: #{ex.message}"
-        end
-      end
+      log_error "EXCEPTION: #{ex.message}", ex.backtrace
     end
 
-    def create_conchita_thread
-      Thread.new do
+    # def create_worker_thread wid, pool
+    #   Thread.new wid do
+    #     begin
+    #       worker = JobWorker.new wid, pool
+    #       #log_info "JobWorker [#{wid}][#{pool}]: #{worker}"
+    #     rescue StandardError => ex
+    #       log_error "JobWorker EXCEPTION: #{ex.message} #{e.backtrace}"
+    #     end
+    #   end
+    # end
+
+    def create_thread wid, klass, pool = nil
+      # Spawn thread and add it to my index
+      log_info "spawning #{klass.name} [#{wid}] [#{pool}]"
+      @workers[wid] = Thread.new do
         begin
-          worker = ConchitaWorker.new :conchita
-          log_info "ConchitaWorker: #{worker}"
+          worker = klass.new wid, pool
         rescue StandardError => ex
-          log_error "ConchitaWorker EXCEPTION: #{ex.message}"
+          log_error "#{klass.name} EXCEPTION: #{ex.message}"
         end
       end
     end
 
-    if Settings.newrelic_enabled?
-      add_transaction_tracer :create_conchita_thread,     category: :task
-      add_transaction_tracer :create_worker_thread,       category: :task
+    # NewRelic instrumentation
+    if Conf.newrelic_enabled?
+      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+      # add_transaction_tracer :create_conchita_thread,     category: :task
+      add_transaction_tracer :create_thread,       category: :task
     end
 
   end