rest-ftp-daemon 0.250.5 → 0.300.1

data/lib/rest-ftp-daemon/job_queue.rb

@@ -2,16 +2,10 @@ module RestFtpDaemon
 
   # Queue that stores all the Jobs waiting to be processed or fully processed
   class JobQueue
-    include LoggerHelper
+    include Shared::LoggerHelper
     attr_reader :logger
-
-    #attr_reader :queues
     attr_reader :jobs
 
-    if Settings.newrelic_enabled?
-      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
-    end
-
     def initialize
       # Instance variables
       @queues = {}
@@ -45,10 +39,18 @@ module RestFtpDaemon
       @queues
     end
 
+    def queued_by_pool
+      result = {}
+      @queues.each do |pool, jobs|
+        result[pool] = jobs.count
+      end
+      result
+    end
+
     def rate_by method_name
       # Init
       result = {}
-      return unless Job.new(0, {}).respond_to? method_name
+      #return unless Job.new(0, {}).respond_to? method_name
 
       # Select only running jobs
       @jobs.each do |job|
@@ -70,9 +72,7 @@ module RestFtpDaemon
 
         # Add its current rate
         result[group] ||= 0
-        #log_info "  2: #{result.inspect} (rate: #{rate})"
         result[group] += rate
-        #log_info "  3: #{result.inspect}"
       end
 
       # Return the rate
@@ -109,7 +109,6 @@ module RestFtpDaemon
       log_info "find_by_id (#{id}, #{prefixed}) > #{id}"
 
       # Search in jobs queues
-      #@jobs.reverse.find { |item| item.id == id }
       @jobs.find { |item| item.id == id }
     end
 
@@ -201,7 +200,7 @@ module RestFtpDaemon
       @mutex.synchronize do
         # Delete jobs from the queue when they match status and age limits
         @jobs.delete_if do |job|
-          # log_info "testing job [#{job.id}] updated_at [#{job.updated_at}]"
+          # log_debug "testing job [#{job.id}] updated_at [#{job.updated_at}]"
 
           # Skip if wrong status, updated_at invalid, or updated since time_limit
           next unless job.status == status
@@ -213,7 +212,7 @@ module RestFtpDaemon
 
           # From any queues, remove it
           @queues.each do |pool, jobs|
-            log_info "#{LOG_INDENT}unqueued from [#{pool}]" if jobs.delete(job)
+            log_debug "#{LOG_INDENT}unqueued from [#{pool}]" if jobs.delete(job)
           end
 
           # Remember we have to delete the original job !
@@ -225,11 +224,19 @@ module RestFtpDaemon
 
   protected
 
+    def log_prefix
+     [nil, nil, nil]
+    end
+
     def prefixed_id id
       "#{@prefix}.#{id}"
     end
 
-    if Settings.newrelic_enabled?
+  private
+
+    # NewRelic instrumentation
+    if Conf.newrelic_enabled?
+      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
       add_transaction_tracer :push,                 category: :task
       add_transaction_tracer :pop,                  category: :task
       add_transaction_tracer :expire,               category: :task

data/lib/rest-ftp-daemon/launcher.rb

@@ -0,0 +1,26 @@
+module RestFtpDaemon
+  class Launcher
+    LAUNCHER_PORT_TIMEOUT       = 3
+    LAUNCHER_PORT_LOCALHOST     = "127.0.0.1"
+
+    class << self
+
+      def local_port_used? port
+        Timeout.timeout(LAUNCHER_PORT_TIMEOUT) do
+          begin
+            TCPSocket.new(LAUNCHER_PORT_LOCALHOST, port).close
+            true
+          rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
+            false
+          rescue Errno::EADDRNOTAVAIL
+            "local_port_used: Errno::EADDRNOTAVAIL"
+          end
+        end
+      rescue Timeout::Error
+        false
+      end
+
+    end
+
+  end
+end

data/lib/rest-ftp-daemon/logger_pool.rb

@@ -16,29 +16,19 @@ module RestFtpDaemon
 
     def create pipe
       # Compute file path / STDERR
-      logfile = Settings.logs[pipe] if Settings.logs.is_a? Hash
+      logfile = Conf[:logs][pipe] if Conf[:logs].is_a? Hash
       logfile ||= STDERR
 
+      # Check if we can write to than file
+      unless File.writable?(logfile)
+        puts "LoggerPool [#{pipe}] logging to hyperspace (cannot write to #{logfile})"
+        logfile = nil
+      end
+
       # Create the logger and return it
       logger = Logger.new(logfile, LOG_ROTATION)   #, 10, 1024000)
-      logger.progname = pipe.to_s.upcase
-
-      # And the formatter
-      logger.formatter = proc do |severity, datetime, progname, messages|
-        # Build common line prefix
-        prefix = LOG_FORMAT_PREFIX % [
-          datetime.strftime(LOG_FORMAT_TIME),
-          severity,
-          progname,
-        ]
-
-        # If we have a bunch of lines, prefix them and send them together
-        if messages.is_a? Array
-          messages.map { |line| prefix + line + LOG_NEWLINE}.join
-        else
-          prefix + messages.to_s + LOG_NEWLINE
-        end
-      end
+      logger.progname = pipe.to_s.downcase
+      logger.formatter = Shared::LoggerFormatter
 
       # Finally return this logger
       logger

data/lib/rest-ftp-daemon/metrics.rb

@@ -0,0 +1,41 @@
+module RestFtpDaemon
+  class Metrics
+
+    def self.sample
+      # Prepare external deps
+      mem = GetProcessMem.new
+
+      # Build final value
+      return  {
+        system: {
+          uptime:           (Time.now - Conf.app_started).round(1),
+          memory:           mem.bytes.to_i,
+          threads:          Thread.list.count,
+          },
+        jobs_by_status:     $queue.jobs_by_status,
+        rate_by_pool:       $queue.rate_by(:pool),
+        rate_by_targethost: $queue.rate_by(:targethost),
+        queued_by_pool:     $queue.queued_by_pool,
+        workers_by_status:  self.workers_count_by_status,
+        }
+    end
+
+    private
+
+      # Collect: workers by status
+      def self.workers_count_by_status
+        # Init
+        counts = {}
+
+        $pool.worker_variables.group_by do |wid, vars|
+          vars[:status]
+        end.each do |status, workers|
+          counts[status] = workers.count
+        end
+
+        # Return count
+        counts
+      end
+
+  end
+end

data/lib/rest-ftp-daemon/notification.rb

@@ -2,7 +2,7 @@ module RestFtpDaemon
 
   # Handles a notification POST using a dedicated thread
   class Notification
-    include LoggerHelper
+    include Shared::LoggerHelper
     attr_reader :logger
 
     attr_accessor :job_id
@@ -35,10 +35,10 @@ module RestFtpDaemon
     def process
       # Check context
       if @url.nil?
-        log_info "skipping (missing url): #{@params.inspect}"
+        log_error "skipping (missing url)", params
         return
       elsif @params[:event].nil?
-        log_info "skipping (missing event): #{@params.inspect}"
+        log_error "skipping (missing event)", params
         return
       end
 
@@ -47,7 +47,7 @@ module RestFtpDaemon
         id:       @params[:id].to_s,
         signal:   "#{NOTIFY_PREFIX}.#{@params[:event]}",
         error:    @params[:error],
-        host:     Settings.host.to_s,
+        host:     Conf.host.to_s,
         }
       flags[:status] = @params[:status] if @params[:status].is_a? Enumerable
       flags[:message] = @params[:message].to_s unless @params[:message].nil?
@@ -64,7 +64,7 @@ module RestFtpDaemon
       headers = {
         "Content-Type"  => "application/json",
         "Accept"        => "application/json",
-        "User-Agent"    => NOTIFY_USERAGENT
+        "User-Agent"    => "#{Conf.app_name}/v#{Conf.app_ver}"
          }
       data = flags.to_json
 
@@ -96,11 +96,8 @@ module RestFtpDaemon
         log_error "UNHANDLED EXCEPTION: #{ex.inspect}"
     end
 
-    def log_context
-      {
-      id: @id,
-      jid: @jid
-      }
+    def log_prefix
+     [nil, @jid, @id]
     end
 
   end

data/lib/rest-ftp-daemon/remote.rb

@@ -2,13 +2,13 @@ module RestFtpDaemon
 
   # Handles transfers for Job class
   class Remote
-    include LoggerHelper
+    include Shared::LoggerHelper
     attr_reader :logger
-    attr_reader :log_context
+    attr_reader :log_prefix
 
-    def initialize url, log_context, options = {}
+    def initialize url, log_prefix, options = {}
       # Logger
-      @log_context = log_context || {}
+      @log_prefix = log_prefix || {}
       @logger = RestFtpDaemon::LoggerPool.instance.get :jobs
 
       # Extract URL parts

data/lib/rest-ftp-daemon/remote_ftp.rb

@@ -6,12 +6,12 @@ module RestFtpDaemon
   class RemoteFTP < Remote
     attr_reader :ftp
 
-    def initialize url, log_context, options = {}
+    def initialize url, log_prefix, options = {}
       # Call super
       super
 
       # Use debug ?
-      @debug = (Settings.at :debug, :ftp) == true
+      @debug = (Conf.at :debug, :ftp) == true
 
       # Create FTP object
       if options[:ftpes]
@@ -26,7 +26,7 @@ module RestFtpDaemon
       @chunk_size = DEFAULT_FTP_CHUNK.to_i * 1024
 
       # Announce object
-      log_info "RemoteFTP.initialize chunk_size:#{@chunk_size}"
+      log_debug "RemoteFTP.initialize chunk_size:#{@chunk_size}"
     end
 
     def connect
@@ -40,7 +40,7 @@ module RestFtpDaemon
 
     def present? target
       size = @ftp.size target.full
-      log_info "RemoteFTP.present? [#{target.name}]"
+      log_debug "RemoteFTP.present? [#{target.name}]"
 
     rescue Net::FTPPermError
       return false
@@ -49,16 +49,16 @@ module RestFtpDaemon
     end
 
     def remove! target
-      log_info "RemoteFTP.remove! [#{target.name}]"
+      log_debug "RemoteFTP.remove! [#{target.name}]"
       @ftp.delete target.full
     rescue Net::FTPPermError
-      log_info "#{LOG_INDENT}[#{target.name}] file not found"
+      log_debug "#{LOG_INDENT}[#{target.name}] file not found"
     else
-      log_info "#{LOG_INDENT}[#{target.name}] removed"
+      log_debug "#{LOG_INDENT}[#{target.name}] removed"
     end
 
     def mkdir directory
-      log_info "RemoteFTP.mkdir [#{directory}]"
+      log_debug "RemoteFTP.mkdir [#{directory}]"
       @ftp.mkdir directory
 
       rescue
@@ -67,7 +67,7 @@ module RestFtpDaemon
 
     def chdir_or_create directory, mkdir = false
       # Init, extract my parent name and my own name
-      log_info "RemoteFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
+      log_debug "RemoteFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
       parent, _current = Helpers.extract_parent(directory)
 
       # Access this directory
@@ -98,7 +98,7 @@ module RestFtpDaemon
       destination.name = tempname if tempname
 
       # Do the transfer
-      log_info "RemoteFTP.push to [#{destination.name}]"
+      log_debug "RemoteFTP.push to [#{destination.name}]"
 
       @ftp.putbinaryfile source.full, target.name, @chunk_size do |data|
         # Update job status after this block transfer

data/lib/rest-ftp-daemon/remote_sftp.rb

@@ -5,21 +5,21 @@ module RestFtpDaemon
   class RemoteSFTP < Remote
     attr_reader :sftp
 
-    def initialize url, log_context, options = {}
+    def initialize url, log_prefix, options = {}
       # Call super
       super
 
       # Use debug ?
-      @debug = (Settings.at :debug, :sftp) == true
+      @debug = (Conf.at :debug, :sftp) == true
 
       # Announce object
-      log_info "RemoteSFTP.initialize"
+      log_debug "RemoteSFTP.initialize"
     end
 
     def connect
       # Connect init
       super
-      log_info "RemoteSFTP.connect [#{@url.user}]@[#{@url.host}]:[#{@url.port}]"
+      log_debug "RemoteSFTP.connect [#{@url.user}]@[#{@url.host}]:[#{@url.port}]"
 
       # Debug level
       verbosity = @debug ? Logger::INFO : false
@@ -35,7 +35,7 @@ module RestFtpDaemon
     end
 
     def present? target
-      log_info "RemoteSFTP.present? [#{target.name}]"
+      log_debug "RemoteSFTP.present? [#{target.name}]"
       stat = @sftp.stat! target.full
 
     rescue Net::SFTP::StatusException
@@ -44,23 +44,18 @@ module RestFtpDaemon
       return stat.size
     end
 
-    # def remove target
-    #   log_info "RemoteSFTP.remove [#{target.name}]"
-    #   @sftp.remove target.full
-    # end
-
     def remove! target
-      log_info "RemoteSFTP.remove! [#{target.name}]"
+      log_debug "RemoteSFTP.remove! [#{target.name}]"
       @sftp.remove target.full
 
     rescue Net::SFTP::StatusException
-      log_info "#{LOG_INDENT}[#{target.name}] file not found"
+      log_debug "#{LOG_INDENT}[#{target.name}] file not found"
     else
-      log_info "#{LOG_INDENT}[#{target.name}] removed"
+      log_debug "#{LOG_INDENT}[#{target.name}] removed"
     end
 
     def mkdir directory
-      log_info "RemoteSFTP.mkdir [#{directory}]"
+      log_debug "RemoteSFTP.mkdir [#{directory}]"
       @sftp.mkdir! directory
 
       rescue
@@ -69,12 +64,12 @@ module RestFtpDaemon
 
     def chdir_or_create directory, mkdir = false
       # Init, extract my parent name and my own name
-      log_info "RemoteSFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
+      log_debug "RemoteSFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
       parent, _current = Helpers.extract_parent(directory)
 
       # Access this directory
       begin
-        # log_info "   chdir [/#{directory}]"
+        log_debug "chdir [/#{directory}]"
         @sftp.opendir! "./#{directory}"
 
       rescue Net::SFTP::StatusException => _e
@@ -97,12 +92,6 @@ module RestFtpDaemon
       raise JobTargetShouldBeDirectory
     end
 
-    # def dir_contents directory
-    #   # Access this directory
-    #   handle = @sftp.opendir! directory
-    #   @sftp.readdir! handle
-    # end
-
     def push source, target, tempname = nil, &callback
       # Push init
       raise RestFtpDaemon::JobAssertionFailed, "push/1" if @sftp.nil?
@@ -112,7 +101,7 @@ module RestFtpDaemon
       destination.name = tempname if tempname
 
       # Do the transfer
-      log_info "RemoteSFTP.push [#{destination.full}]"
+      log_debug "RemoteSFTP.push [#{destination.full}]"
       @sftp.upload! source.full, destination.full do |event, _uploader, *args|
         case event
         when :open then
@@ -140,7 +129,7 @@ module RestFtpDaemon
 
       # Rename if needed
       if tempname
-        log_info "RemoteSFTP.push rename to\t[#{target.name}]"
+        log_debug "RemoteSFTP.push rename to\t[#{target.name}]"
         @sftp.rename! destination.full, target.full, flags
       end