rest-ftp-daemon 0.214.0 → 0.220.0

data/lib/rest-ftp-daemon/job_queue.rb

@@ -1,9 +1,12 @@
 module RestFtpDaemon
-  class JobQueue < Queue
+  class JobQueue
+    include LoggerHelper
+    attr_reader :logger
 
     attr_reader :queue
     attr_reader :jobs
 
+
     if Settings.newrelic_enabled?
       include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
     end
@@ -24,7 +27,7 @@ module RestFtpDaemon
       # Identifiers generator
       @last_id = 0
       @prefix = Helpers.identifier JOB_IDENT_LEN
-      info "queue initialized with prefix: #{@prefix}"
+      log_info "JobQueue initialized (prefix: #{@prefix})"
 
       # Mutex for counters
       @counters = {}
@@ -32,9 +35,7 @@ module RestFtpDaemon
     end
 
     def generate_id
-      # rand(36**8).to_s(36)
       @mutex.synchronize do
-        @last_id ||= 0
         @last_id += 1
       end
       prefixed_id @last_id
@@ -100,7 +101,7 @@ module RestFtpDaemon
     def find_by_id id, prefixed = false
       # Build a prefixed id if expected
       id = prefixed_id(id) if prefixed
-      info "find_by_id (#{id}, #{prefixed}) > #{id}"
+      log_info "find_by_id (#{id}, #{prefixed}) > #{id}"
 
       # Search in jobs queues
       @jobs.select { |item| item.id == id }.last
@@ -184,8 +185,8 @@ module RestFtpDaemon
           next if job.updated_at > before
 
           # Ok, we have to clean it up ..
-          info "expire [#{status.to_s}] [#{maxage}] > [#{job.id}] [#{job.updated_at}]"
-          info "       + unqueued" if @queue.delete(job)
+          log_info "expire [#{status.to_s}] [#{maxage}] > [#{job.id}] [#{job.updated_at}]"
+          log_info "       + unqueued" if @queue.delete(job)
 
           true
         end
@@ -206,21 +207,13 @@ module RestFtpDaemon
       end
     end
 
-    def info message, lines = []
-      return if @logger.nil?
-
-      # Forward to logger
-      @logger.info_with_id message,
-        id: @id,
-        lines: lines,
-        origin: self.class.to_s
-    end
-
     if Settings.newrelic_enabled?
       add_transaction_tracer :push, :category => :task
       add_transaction_tracer :pop, :category => :task
       add_transaction_tracer :sort_queue!, :category => :task
       add_transaction_tracer :expire, :category => :task
+      add_transaction_tracer :counts_by_status, :category => :task
+      add_transaction_tracer :filter_jobs, :category => :task
     end
 
   end

data/lib/rest-ftp-daemon/logger.rb

@@ -1,29 +1,70 @@
 class Logger
 
   def info_with_id message, context = {}
-    # Ensure context is a hash of options
+    # Ensure context is a hash of options and init
     context = {} unless context.is_a? Hash
+    context[:level] ||= Logger::DEBUG
 
-    # Default context
-    #add Logger::DEBUG,  "info_with_id/context: #{context.inspect} | #{message}"
-    context[:level] ||= 0
+    # Build prefixes depending on this context
+    prefix1 = build_prefix(context)
+    prefix2 = build_prefix() + '   | '
 
-    # Common message header
-    field_wid = "%#{-LOG_COL_WID.to_i}s" % context[:wid].to_s
-    field_jid = "%#{-LOG_COL_JID.to_i}s" % context[:jid].to_s
-    field_id = "%#{-LOG_COL_ID.to_i}s" % context[:id].to_s
-    prefix = "#{field_wid} \t#{field_jid} \t#{field_id}\t#{'  '*(context[:level].to_i+1)}"
+    # # Build output lines
+    # output = []
+    # output << prefix1 + message.strip
 
-    # Send main message
-    add Logger::INFO, prefix + message.to_s
+    # # Add optional lines
+    # context[:lines].each do |line|
+    #   # line.strip!
+    #   # next if line.empty?
+    #   output << prefix2 + line[0..LOG_TRIM_LINE]
+    # end if context[:lines].is_a? Enumerable
 
-    # Dump context lines if provided
-    context[:lines].each do |line|
-      line.strip!
-      next if line.empty?
-      add Logger::INFO, prefix + '   | ' + line[0..LOG_TRIM_LINE]
-    end if context[:lines].is_a? Enumerable
+    # Use "context[:lines]" according to its type
+    lines = context[:lines]
 
+    if lines.is_a? Hash
+      output = build_from_hash prefix2, lines
+
+    elsif lines.is_a? Array
+      output = build_from_array prefix2, lines
+
+    else
+      output = []
+
+    end
+
+
+    # Prepend plain message to output
+    output.unshift (prefix1 + message.strip)
+
+    # Send all this to logger
+    add context[:level], output
   end
 
+  def build_prefix context = {}
+    LOG_FORMAT_MESSAGE % [
+      context[:wid].to_s,
+      context[:jid].to_s,
+      context[:id].to_s,
+      context[:level].to_i+1,
+    ]
+  end
+
+  protected
+
+    def build_from_array prefix, lines
+      lines.map do |value|
+        text = value.to_s.strip[0..LOG_TRIM_LINE]
+        "#{prefix}#{text}"
+      end
+    end
+
+    def build_from_hash prefix, lines
+      lines.map do |name, value|
+        text = value.to_s.strip[0..LOG_TRIM_LINE]
+        "#{prefix}#{name}: #{text}"
+      end
+    end
+
 end

data/lib/rest-ftp-daemon/logger_helper.rb

@@ -0,0 +1,26 @@
+module RestFtpDaemon
+  module LoggerHelper
+
+  protected
+
+    def log_info message, lines = []
+      logger.info_with_id message, log_context.merge({
+        from: self.class.to_s,
+        lines: lines,
+        level: Logger::INFO
+        })
+    end
+
+    def log_error message, lines = []
+      logger.info_with_id message, log_context.merge({
+        from: self.class.to_s,
+        lines: lines,
+        level: Logger::ERROR
+        })
+    end
+
+    def log_context
+      {}
+    end
+  end
+end

data/lib/rest-ftp-daemon/logger_pool.rb

@@ -1,6 +1,3 @@
-require 'singleton'
-require 'logger'
-
 module RestFtpDaemon
   class LoggerPool
     include Singleton
@@ -19,26 +16,29 @@ module RestFtpDaemon
       logfile ||= STDERR
 
       # Create the logger and return it
-      logger = Logger.new(logfile, 'daily')   #, 10, 1024000)
+      logger = Logger.new(logfile, LOG_ROTATION)   #, 10, 1024000)
       logger.progname = pipe.to_s.upcase
 
       # And the formatter
-      logger.formatter = proc do |severity, datetime, progname, message|
-        # stamp = datetime.strftime("%Y-%m-%d %H:%M:%S")
-        # field_pipe = "%-#{LOG_PIPE_LEN.to_i}s" % progname
-        # "#{stamp}\t#{field_pipe}\t#{message}\n"
-        "%s\t%-#{LOG_PIPE_LEN.to_i}s\t%s\n" % [
-          datetime.strftime("%Y-%m-%d %H:%M:%S"),
+      logger.formatter = proc do |severity, datetime, progname, messages|
+        # Build common line prefix
+        prefix = LOG_FORMAT_PREFIX % [
+          datetime.strftime(LOG_FORMAT_TIME),
+          severity,
           progname,
-          message,
         ]
+
+        # If we have a bunch of lines, prefix them and send them together
+        if messages.is_a? Array
+          messages.map { |line| prefix + line + LOG_NEWLINE}.join
+        else
+          prefix + messages.to_s + LOG_NEWLINE
+        end
       end
 
       # Finally return this logger
       logger
     end
 
-  private
-
   end
 end

data/lib/rest-ftp-daemon/notification.rb

@@ -1,5 +1,8 @@
 module RestFtpDaemon
   class Notification
+    include LoggerHelper
+    attr_reader :logger
+
     attr_accessor :job_id
     attr_accessor :signal
     attr_accessor :error
@@ -18,10 +21,10 @@ module RestFtpDaemon
 
       # Check context
       if url.nil?
-        info "skipping (missing url): #{params.inspect}"
+        log_info "skipping (missing url): #{params.inspect}"
         return
       elsif params[:event].nil?
-        info "skipping (missing event): #{params.inspect}"
+        log_info "skipping (missing event): #{params.inspect}"
         return
       end
 
@@ -34,7 +37,7 @@ module RestFtpDaemon
         }
       body[:status] = params[:status] if params[:status].is_a? Enumerable
       @jid = params[:id]
-      info "initialized"
+      log_info "initialized"
 
 
       # Send message in a thread
@@ -47,7 +50,7 @@ module RestFtpDaemon
           'User-Agent'    => "#{APP_NAME} - #{APP_VER}"
            }
         data = body.to_json
-        info "sending #{data}"
+        log_info "sending #{data}"
 
 
         # Prepare HTTP client
@@ -63,9 +66,9 @@ module RestFtpDaemon
         if response_lines.size > 1
           human_size = Helpers.format_bytes(response.body.bytesize, "B")
           #human_size = 0
-          info "received [#{response.code}] #{human_size} (#{response_lines.size} lines)", response_lines
+          log_info "received [#{response.code}] #{human_size} (#{response_lines.size} lines)", response_lines
         else
-          info "received [#{response.code}] #{response.body.strip}"
+          log_info "received [#{response.code}] #{response.body.strip}"
         end
 
       end
@@ -74,14 +77,11 @@ module RestFtpDaemon
 
   protected
 
-    def info message, lines = []
-      return if @logger.nil?
-
-      @logger.info_with_id message,
-        id: @id,
-        jid: @jid,
-        lines: lines,
-        origin: self.class.to_s
+    def log_context
+      {
+      id: @id,
+      jid: @jid
+      }
     end
 
   end

data/lib/rest-ftp-daemon/settings.rb

@@ -25,7 +25,7 @@ class Settings < Settingslogic
   end
 
   def newrelic_enabled?
-    Settings.newrelic.is_a?(Hash) && Settings.at(:newrelic, :license)
+    Settings.at(:debug, :newrelic)
   end
 
   def init_newrelic
@@ -38,10 +38,10 @@ class Settings < Settingslogic
     #Settings['newrelic']['enabled'] = true
 
     # License
-    ENV['NEW_RELIC_LICENSE_KEY'] = Settings.at(:newrelic, :license)
+    ENV['NEW_RELIC_LICENSE_KEY'] = Settings.at(:debug, :newrelic)
 
     # Appname
-    ENV['NEW_RELIC_APP_NAME'] = Settings.at(:newrelic, :appname) || "#{APP_NICK}-#{Settings.host}-#{APP_ENV}"
+    ENV['NEW_RELIC_APP_NAME'] = "#{APP_NICK}-#{Settings.host}-#{APP_ENV}"
 
     # Logfile
     ENV['NEW_RELIC_LOG'] = Settings.at(:logs, :newrelic)

data/lib/rest-ftp-daemon/worker.rb

@@ -0,0 +1,58 @@
+module RestFtpDaemon
+  class Worker
+    include LoggerHelper
+    attr_reader :logger
+
+    if Settings.newrelic_enabled?
+      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+    end
+
+    def initialize wid
+      # Logger
+      @logger = RestFtpDaemon::LoggerPool.instance.get :workers
+
+      # Worker name
+      @wid = wid
+
+      # Set thread context
+      Thread.current.thread_variable_set :wid, wid
+      Thread.current.thread_variable_set :started_at, Time.now
+      worker_status :starting
+    end
+
+  protected
+
+    def log_context
+      {
+      wid: @wid,
+      tag_1_worker_object: true
+      }
+    end
+
+    def start
+      loop do
+        begin
+          work
+        rescue Exception => e
+          log_error "WORKER EXCEPTION: #{e.inspect}"
+          sleep 1
+        end
+      end
+    end
+
+    def worker_status status
+      Thread.current.thread_variable_set :status, status
+      Thread.current.thread_variable_set :updted_at, Time.now
+    end
+
+    def worker_jid jid
+      Thread.current.thread_variable_set :jid, jid
+      Thread.current.thread_variable_set :updted_at, Time.now
+    end
+
+    if Settings.newrelic_enabled?
+      add_transaction_tracer :work, :category => :task
+    end
+
+  end
+end

data/lib/rest-ftp-daemon/worker_conchita.rb

@@ -0,0 +1,49 @@
+module RestFtpDaemon
+  class ConchitaWorker < Worker
+
+    def initialize wid = :conchita
+      # Generic worker initialize
+      super
+
+      # Conchita configuration
+      @conchita = Settings.conchita
+      if !@conchita.is_a? Hash
+        return log_info "ConchitaWorker: missing conchita.* configuration"
+      elsif @conchita[:timer].nil?
+        return log_info "ConchitaWorker: missing conchita.timer value"
+      end
+
+      # Start main loop
+      log_info "ConchitaWorker starting", @conchita
+      start
+    end
+
+  protected
+
+    def work
+      worker_status :cleaning
+
+      # Cleanup queues according to configured max-age
+      $queue.expire JOB_STATUS_FINISHED,  maxage(JOB_STATUS_FINISHED)
+      $queue.expire JOB_STATUS_FAILED,    maxage(JOB_STATUS_FAILED)
+      $queue.expire JOB_STATUS_QUEUED,    maxage(JOB_STATUS_QUEUED)
+
+      # Force garbage collector
+      worker_status :collecting
+      GC.start if @conchita["garbage_collector"]
+
+    rescue Exception => e
+      log_error "EXCEPTION: #{e.inspect}"
+      sleep 1
+    else
+      # Sleep for a few seconds
+      worker_status :sleeping
+      sleep @conchita[:timer]
+    end
+
+    def maxage status
+      @conchita["clean_#{status.to_s}"] || 0
+    end
+
+  end
+end

data/lib/rest-ftp-daemon/worker_job.rb

@@ -0,0 +1,64 @@
+module RestFtpDaemon
+  class JobWorker < Worker
+
+    def initialize wid
+      # Generic worker initialize
+      super
+
+      # Timeout config
+      @timeout = (Settings.transfer.timeout rescue nil) || DEFAULT_WORKER_TIMEOUT
+
+      # Start main loop
+      log_info "JobWorker starting", ["timeout: #{@timeout}"]
+      start
+    end
+
+  protected
+
+    def work
+      # Wait for a job to come into the queue
+      worker_status :waiting
+      log_info "waiting for a job"
+      job = $queue.pop
+
+      # Prepare the job for processing
+      worker_status :working
+      worker_jid job.id
+      log_info "working with job [#{job.id}]"
+      job.wid = Thread.current.thread_variable_get :wid
+
+      # Processs this job protected by a timeout
+      Timeout::timeout(@timeout, RestFtpDaemon::JobTimeout) do
+        job.process
+      end
+
+      # Processing done
+      worker_status :finished
+      log_info "finished with job [#{job.id}]"
+      worker_jid nil
+      job.wid = nil
+
+      # Increment total processed jobs count
+      $queue.counter_inc :jobs_processed
+
+    rescue RestFtpDaemon::JobTimeout => ex
+      log_error "JOB TIMED OUT", lines: ex.backtrace
+      worker_status :timeout
+      job.oops_you_stop_now ex unless job.nil?
+      sleep 1
+
+    rescue Exception => ex
+      log_error "JOB UNHDNALED EXCEPTION: #{ex.message}", lines: ex.backtrace
+      worker_status :crashed
+      job.oops_after_crash ex unless job.nil?
+      sleep 1
+
+    else
+      # Clean job status
+      worker_status :free
+      job.wid = nil
+
+    end
+
+  end
+end