rest-ftp-daemon 0.250.5 → 0.300.1

data/lib/rest-ftp-daemon.rb

@@ -9,25 +9,31 @@ require "syslog"
 require "net/http"
 require "thread"
 require "singleton"
-require "newrelic_rpm"
 require "grape"
 require "grape-entity"
+require "newrelic_rpm"
+
+
+# Shared libs
+require_relative "shared/logger_formatter"
+require_relative "shared/logger_helper"
+require_relative "shared/conf"
+
 
 # Project's libs
 require_relative "rest-ftp-daemon/constants"
 require_relative "rest-ftp-daemon/array"
-require_relative "rest-ftp-daemon/settings"
 require_relative "rest-ftp-daemon/exceptions"
 require_relative "rest-ftp-daemon/helpers"
-require_relative "rest-ftp-daemon/logger_helper"
 require_relative "rest-ftp-daemon/logger_pool"
-require_relative "rest-ftp-daemon/logger"
+require_relative "rest-ftp-daemon/metrics"
 require_relative "rest-ftp-daemon/paginate"
 require_relative "rest-ftp-daemon/uri"
 require_relative "rest-ftp-daemon/job_queue"
 require_relative "rest-ftp-daemon/counters"
 require_relative "rest-ftp-daemon/worker"
 require_relative "rest-ftp-daemon/worker_conchita"
+require_relative "rest-ftp-daemon/worker_reporter"
 require_relative "rest-ftp-daemon/worker_job"
 require_relative "rest-ftp-daemon/worker_pool"
 require_relative "rest-ftp-daemon/job"

data/lib/rest-ftp-daemon/api/config.rb

@@ -2,6 +2,7 @@ module RestFtpDaemon
   module API
     class Config < Grape::API
 
+      ### ENDPOINTS
       desc "Show daemon config"
       get "/" do
         status 200
@@ -10,8 +11,8 @@ module RestFtpDaemon
 
       desc "Reload daemon config"
       post "/reload" do
-        if Settings.at(:debug, :allow_reload)==true
-          Settings.reload!
+        if Conf.at(:debug, :allow_reload)==true
+          Conf.reload!
           status 200
           return Helpers.get_censored_config
         else

data/lib/rest-ftp-daemon/api/dashboard.rb

@@ -5,14 +5,12 @@ require "facter"
 
 module RestFtpDaemon
   module API
-
     class Dashbaord < Grape::API
 
       ### HELPERS
-
       helpers do
         def render name, values={}
-          template = File.read("#{APP_LIBS}/views/#{name}.haml")
+          template = File.read("#{Conf.app_libs}/views/#{name}.haml")
 
           haml_engine = Haml::Engine.new(template, encoding: Encoding::UTF_8)
               #:encoding => Encoding::ASCII_8BIT
@@ -56,7 +54,6 @@ module RestFtpDaemon
 
       end
 
-
       ### DASHBOARD
       desc "Show a global dashboard"
       get "/" do

data/lib/rest-ftp-daemon/api/debug.rb

@@ -2,33 +2,46 @@ module RestFtpDaemon
   module API
     class Debug < Grape::API
 
-      desc "Show app routes, params encodings"
-      get "/" do
-        # Encodings
-        encodings = {}
-        jobs = $queue.jobs
+      ### HELPERS
+      helpers do
+
+        def debug_metrics
+          Metrics.sample
+        end
 
-        jobs.each do |job|
-          # here = out[job.id] = {}
-          me = encodings[job.id] = {}
+        def debug_encodings
+          # Encodings
+          encodings = {}
+          jobs = $queue.jobs
 
-          me[:error] = job.error.encoding.to_s unless job.error.nil?
-          me[:status] = job.status.encoding.to_s unless job.status.nil?
+          jobs.each do |job|
+            # here = out[job.id] = {}
+            me = encodings[job.id] = {}
 
-          Job::FIELDS.each do |name|
-            value = job.send(name)
-            me[name] = value.encoding.to_s if value.is_a? String
-          end
+            me[:error] = job.error.encoding.to_s unless job.error.nil?
+            me[:status] = job.status.encoding.to_s unless job.status.nil?
 
-          job.infos.each do |name, value|
-            me["infos_#{name}"] = value.encoding.to_s if value.is_a? String
+            Job::FIELDS.each do |name|
+              value = job.send(name)
+              me[name] = value.encoding.to_s if value.is_a? String
+            end
+
+            job.infos.each do |name, value|
+              me["infos_#{name}"] = value.encoding.to_s if value.is_a? String
+            end
           end
         end
 
+      end
+
+      ### ENDPOINTS
+      desc "debug"
+      get "/" do
        # Build response
        return  {
+          metrics: debug_metrics,
           routes: RestFtpDaemon::API::Root.routes,
-          encodings: encodings,
+          encodings: debug_encodings,
           }
       end
 

data/lib/rest-ftp-daemon/api/job_presenter.rb

@@ -1,7 +1,6 @@
 module RestFtpDaemon
   module API
     module Entities
-
       class JobPresenter < Grape::Entity
         # Job ID
         expose :id
@@ -30,7 +29,6 @@ module RestFtpDaemon
         expose :infos, unless: :hide_infos
 
       end
-
     end
   end
 end

data/lib/rest-ftp-daemon/api/jobs.rb

@@ -2,6 +2,7 @@ module RestFtpDaemon
   module API
     class Jobs < Grape::API
 
+      ### ENDPOINTS
       desc "Read job with ID"
       params do
         requires :id, type: String, desc: "ID of the Job to read"
@@ -56,15 +57,15 @@ module RestFtpDaemon
         optional :overwrite,
           type: Boolean,
           desc: "Overwrites files at target server",
-          default: Settings.at(:transfer, :overwrite)
+          default: Conf.at(:transfer, :overwrite)
         optional :mkdir,
           type: Boolean,
           desc: "Create missing directories on target server",
-          default: Settings.at(:transfer, :mkdir)
+          default: Conf.at(:transfer, :mkdir)
         optional :tempfile,
           type: Boolean,
           desc: "Upload to a temp file before renaming it to the target filename",
-          default: Settings.at(:transfer, :tempfile)
+          default: Conf.at(:transfer, :tempfile)
       end
       post "/" do
         # log_debug params.to_json

data/lib/rest-ftp-daemon/api/root.rb

@@ -3,8 +3,11 @@ module RestFtpDaemon
     class Root < Grape::API
 
       ### LOGGING & HELPERS
-
       helpers do
+        def log_prefix
+          ['API', nil, nil]
+        end
+
         def logger
           Root.logger
         end
@@ -18,7 +21,7 @@ module RestFtpDaemon
           request_method = env['REQUEST_METHOD']
           request_path   = env['REQUEST_PATH']
           request_uri    = env['REQUEST_URI']
-          log_info "HTTP #{request_method} #{request_uri}", params
+          log_info       "HTTP #{request_method} #{request_uri}", params
         end
       end
 
@@ -26,10 +29,8 @@ module RestFtpDaemon
         log_request
       end
 
-
       ### CLASS CONFIG
-
-      helpers RestFtpDaemon::LoggerHelper
+      helpers Shared::LoggerHelper
       logger RestFtpDaemon::LoggerPool.instance.get :api
       do_not_route_head!
       do_not_route_options!
@@ -39,7 +40,6 @@ module RestFtpDaemon
 
 
       ### MOUNTPOINTS
-
       mount RestFtpDaemon::API::Status => MOUNT_STATUS
       mount RestFtpDaemon::API::Jobs => MOUNT_JOBS
       mount RestFtpDaemon::API::Dashbaord => MOUNT_BOARD
@@ -48,9 +48,7 @@ module RestFtpDaemon
 
 
       ### INITIALIZATION
-
       def initialize
-        # Call daddy
         super
 
         # Check that Queue and Pool are available
@@ -59,8 +57,7 @@ module RestFtpDaemon
       end
 
 
-      ### ROOT URL ACCESS
-
+      ### ENDPOINTS
       get "/" do
         redirect Helpers.dashboard_filter_url()
       end

data/lib/rest-ftp-daemon/api/status.rb

@@ -4,9 +4,9 @@ module RestFtpDaemon
   module API
     class Status < Grape::API
 
+      ### ENDPOINTS
       desc "Show daemon status"
       get "/" do
-        mem = GetProcessMem.new
         status 200
 
         # Get counters
@@ -23,24 +23,18 @@ module RestFtpDaemon
 
         # Generate sutrcture
         return  {
+          name: Conf.app_name,
+          version: Conf.app_ver,
+          started: Conf.app_started,
           hostname: `hostname`.to_s.chomp,
-          version: APP_VER,
-
-          started: APP_STARTED,
-          uptime: (Time.now - APP_STARTED).round(1),
-
-          memory_bytes: mem.bytes.to_i,
-          memory_mb: mem.mb.round(0),
-
-          status: $queue.jobs_by_status,
           jobs_count: $queue.jobs_count,
 
-          counters: counters,
+          metrics: Metrics.sample,
 
-          rate_by_pool: $queue.rate_by(:pool),
-          rate_by_targethost: $queue.rate_by(:targethost),
+          counters: counters,
 
           workers: $pool.worker_variables,
+
           }
       end
 

data/lib/rest-ftp-daemon/constants.rb

@@ -1,31 +1,11 @@
-# Terrific constants
-APP_NAME = "rest-ftp-daemon"
-APP_NICK = "rftpd"
-APP_VER = "0.250.5"
-
-# Provide default config file information
-APP_LIB = File.expand_path(File.dirname(__FILE__))
-APP_ROOT = File.expand_path(File.dirname(__FILE__) + "/../../")
-
-DEFAULT_CONFIG_PATH = File.expand_path "/etc/#{APP_NAME}.yml"
-SAMPLE_CONFIG_FILE = File.expand_path(File.join File.dirname(__FILE__), "/../../rest-ftp-daemon.yml.sample")
-
-TAIL_MESSAGE = <<EOD
-
-A default configuration is available here: #{SAMPLE_CONFIG_FILE}.
-You should copy it to the expected location #{DEFAULT_CONFIG_PATH}:
-
-sudo cp #{SAMPLE_CONFIG_FILE} #{DEFAULT_CONFIG_PATH}
-EOD
+# Misc constants
 
 
 # Configuration defaults
-# DEFAULT_WORKERS         = 2
 DEFAULT_POOL            = "default"
-DEFAULT_WORKER_TIMEOUT  = 1800  # 1h
 DEFAULT_SFTP_TIMEOUT    = 600   # 10mn
 DEFAULT_FTP_CHUNK       = 1024  # 1 MB
-DEFAULT_PAGE_SIZE       = 80    # 50 lines
+DEFAULT_PAGE_SIZE       = 50    # 50 lines
 DEFAULT_RETRY_DELAY     = 10    # 10s
 
 
@@ -36,6 +16,29 @@ JOB_TEMPFILE_LEN        = 8
 JOB_UPDATE_INTERVAL     = 1
 
 
+# Constants: logger
+LOG_ROTATION             = "daily"
+LOG_FORMAT_PROGNAME     = "%d\t%s"
+
+LOG_HEADER_TIME          = "%Y-%m-%d %H:%M:%S"
+LOG_HEADER_FORMAT        = "%s \t%d\t%-8s %-15s "
+LOG_MESSAGE_TRIM         = 200
+LOG_MESSAGE_TEXT         = "%s%s"
+LOG_MESSAGE_ARRAY        = "%s     - %s"
+LOG_MESSAGE_HASH         = "%s     * %-20s %s"
+
+# Constants: logger app-specific prefix
+LOG_PREFIX_WID           = 8
+LOG_PREFIX_JID           = JOB_IDENT_LEN + 3 + 2
+LOG_PREFIX_ID            = 6
+LOG_PREFIX_FORMAT        = "%#{-LOG_PREFIX_WID.to_i}s %#{-LOG_PREFIX_JID.to_i}s %#{-LOG_PREFIX_ID.to_i}s"
+
+
+# Constants: logger to be cleaned up
+LOG_PIPE_LEN            = 10
+LOG_INDENT              = "\t"
+
+
 # Jobs statuses
 JOB_STATUS_PREPARING    = "preparing"
 JOB_STATUS_RUNNING      = "running"
@@ -73,6 +76,7 @@ WORKER_STATUS_FINISHED  = "finished"
 WORKER_STATUS_TIMEOUT   = "timeout"
 WORKER_STATUS_CRASHED   = "crashed"
 WORKER_STATUS_CLEANING  = "cleaning"
+WORKER_STATUS_REPORTING = "reporting"
 WORKER_STYLES = {
   WORKER_STATUS_WAITING  => nil,
   WORKER_STATUS_RUNNING  => :info,
@@ -81,23 +85,7 @@ WORKER_STYLES = {
   }
 
 
-# Logging and startup
-LOG_PIPE_LEN            = 10
-LOG_COL_WID             = 8
-LOG_COL_JID             = JOB_IDENT_LEN + 3 + 2
-LOG_COL_ID              = 6
-LOG_TRIM_LINE           = 200
-LOG_DUMPS               = File.dirname(__FILE__) + "/../../log/"
-LOG_ROTATION            = "daily"
-LOG_FORMAT_TIME         = "%Y-%m-%d %H:%M:%S"
-LOG_FORMAT_PREFIX       = "%s %s\t%-#{LOG_PIPE_LEN.to_i}s\t"
-LOG_FORMAT_MESSAGE      = "%#{-LOG_COL_WID.to_i}s\t%#{-LOG_COL_JID.to_i}s\t%#{-LOG_COL_ID.to_i}s"
-LOG_NEWLINE             = "\n"
-LOG_INDENT              = "\t"
-BIND_PORT_TIMEOUT       = 3
-BIND_PORT_LOCALHOST     = "127.0.0.1"
-
-ENV_PRODUCTION           = "production"
+# API mountpoints
 MOUNT_JOBS               = "/jobs"
 MOUNT_BOARD              = "/board"
 MOUNT_STATUS             = "/status"
@@ -107,10 +95,4 @@ MOUNT_CONFIG             = "/config"
 
 # Notifications
 NOTIFY_PREFIX           = "rftpd"
-NOTIFY_USERAGENT        = "#{APP_NAME}/v#{APP_VER}"
 NOTIFY_IDENTIFIER_LEN   = 4
-
-
-# Initialize defaults
-APP_STARTED = Time.now
-APP_LIBS = File.dirname(__FILE__)

data/lib/rest-ftp-daemon/counters.rb

@@ -4,10 +4,6 @@ module RestFtpDaemon
   class Counters
     attr_reader :stats
 
-    if Settings.newrelic_enabled?
-      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
-    end
-
     def initialize
       @stats = {}
       @mutex_stats = Mutex.new

data/lib/rest-ftp-daemon/helpers.rb

@@ -2,9 +2,9 @@ module RestFtpDaemon
   class Helpers
 
     def self.get_censored_config
-      config = Settings.to_hash
-      config[:users] = Settings.users.keys if Settings.users
-      config[:endpoints] = Settings.endpoints.keys if Settings.endpoints
+      config = Conf.to_hash
+      config[:users] = Conf[:users].keys if Conf[:users]
+      config[:endpoints] = Conf[:endpoints].keys if Conf[:endpoints]
       config
     end
 
@@ -60,21 +60,6 @@ module RestFtpDaemon
       return m[1], m[2] unless m.nil?
     end
 
-    def self.local_port_used? port
-      Timeout.timeout(BIND_PORT_TIMEOUT) do
-        begin
-          TCPSocket.new(BIND_PORT_LOCALHOST, port).close
-          true
-        rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
-          false
-        rescue Errno::EADDRNOTAVAIL
-          "Settings.local_port_used: Errno::EADDRNOTAVAIL"
-        end
-      end
-    rescue Timeout::Error
-      false
-    end
-
     def self.job_method_label method
       return if method.nil?
       klass = case method

data/lib/rest-ftp-daemon/job.rb

@@ -4,13 +4,9 @@ module RestFtpDaemon
 
   # Reprensents work to be done along with parameters to process it
   class Job
-    include LoggerHelper
+    include Shared::LoggerHelper
     attr_reader :logger
 
-    if Settings.newrelic_enabled?
-      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
-    end
-
     FIELDS = [:source, :target, :label, :priority, :pool, :notify, :overwrite, :mkdir, :tempfile]
 
     attr_accessor :wid
@@ -59,7 +55,7 @@ module RestFtpDaemon
       end
 
       # Set pool
-      pools = (Settings.pools || {})
+      pools = (Conf[:pools] || {})
       # Check if pool name exists
       if (pools.keys.include? params[:pool])
         @pool = params[:pool].to_s
@@ -71,7 +67,7 @@ module RestFtpDaemon
       reset
 
       # Read source file size and parameters
-      @notify_after_sec = Settings.at(:transfer, :notify_after_sec) rescue nil
+      @notify_after_sec = Conf.at(:transfer, :notify_after_sec) rescue nil
     end
 
     def reset
@@ -277,8 +273,8 @@ module RestFtpDaemon
 
     def replace_tokens path
       # Ensure endpoints are not a nil value
-      return path unless Settings.endpoints.is_a? Enumerable
-      vectors = Settings.endpoints.clone
+      return path unless Conf[:endpoints].is_a? Enumerable
+      vectors = Conf[:endpoints].clone
 
       # Stack RANDOM into tokens
       vectors["RANDOM"] = SecureRandom.hex(JOB_RANDOM_LEN)
@@ -328,21 +324,21 @@ module RestFtpDaemon
         # set_info :target, :method, :ftp
         set_info :target, :method, JOB_METHOD_FTP
         #@target_method = :ftp
-        @remote = RemoteFTP.new target_uri, log_context
+        @remote = RemoteFTP.new target_uri, log_prefix
 
       elsif (target_uri.is_a? URI::FTPES) || (target_uri.is_a? URI::FTPS)
         log_info "Job.prepare target_method FTPES"
         # set_info :target, :method, :ftpes
         set_info :target, :method, JOB_METHOD_FTPS
         # @target_method = :ftpes
-        @remote = RemoteFTP.new target_uri, log_context, ftpes: true
+        @remote = RemoteFTP.new target_uri, log_prefix, ftpes: true
 
       elsif target_uri.is_a? URI::SFTP
         log_info "Job.prepare target_method SFTP"
         # set_info :target, :method, :sftp
         set_info :target, :method, JOB_METHOD_SFTP
         # @target_method = :sftp
-        @remote = RemoteSFTP.new target_uri, log_context
+        @remote = RemoteSFTP.new target_uri, log_prefix
 
       else
         log_info "Job.prepare unknown scheme [#{target_uri.scheme}]"
@@ -413,11 +409,8 @@ module RestFtpDaemon
 
   private
 
-    def log_context
-      {
-        wid: @wid,
-        jid: @id,
-      }
+    def log_prefix
+     [@wid, @id, nil]
     end
 
     def find_local path
@@ -509,14 +502,14 @@ module RestFtpDaemon
       tempname = nil
       if @tempfile
         tempname = "#{target.name}.temp-#{Helpers.identifier(JOB_TEMPFILE_LEN)}"
-        #log_info "Job.remote_push tempname [#{tempname}]"
+        log_debug "Job.remote_push tempname [#{tempname}]"
       end
 
       # Remove any existing version if expected, or test its presence
       if @overwrite
         @remote.remove! target
       elsif size = @remote.present?(target)
-        log_info "Job.remote_push existing (#{Helpers.format_bytes size, 'B'})"
+        log_debug "Job.remote_push existing (#{Helpers.format_bytes size, 'B'})"
         raise RestFtpDaemon::JobTargetFileExists
       end
 
@@ -568,7 +561,7 @@ module RestFtpDaemon
         stack << (Helpers.format_bytes @transfer_total, "B")
         stack << (Helpers.format_bytes @current_bitrate.round(0), "bps")
         stack2 = stack.map { |txt| ("%#{LOG_PIPE_LEN.to_i}s" % txt) }.join("\t")
-        log_info "#{LOG_INDENT}progress #{stack2} \t#{name}"
+        log_debug "Job.progress #{stack2} \t#{name}"
 
         # Remember when we last did it
         @progress_at = now
@@ -665,7 +658,9 @@ module RestFtpDaemon
       client_notify event, error: error, status: notif_status, message: "#{exception.class} | #{exception.message}"
     end
 
-    if Settings.newrelic_enabled?
+    # NewRelic instrumentation
+    if Conf.newrelic_enabled?
+      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
       add_transaction_tracer :prepare,        category: :task
       add_transaction_tracer :run,            category: :task
       add_transaction_tracer :client_notify,  category: :task