rest-ftp-daemon 0.302.3 → 0.304.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 404ecae7bf682e2d02aafe69f1a04acfd5792ba6
-  data.tar.gz: c9b5924aa5387ea7423fd9464c695ffc37a3a424
+  metadata.gz: 879a66df142ffdef4e82af5932e0ae6f862382b1
+  data.tar.gz: d81bb16f0c971a2b7a07e0902aa5ab40bce438e7
 SHA512:
-  metadata.gz: 1e88538745df4eca736c1f9b238e6645a6de0fa412b117d741554d7091c9b4d5eeb741bf2be3bc57f9aad5a399b640a163098700c17700b9e40e146cbf231360
-  data.tar.gz: 6e2afd3392eddfbfd6cb3a8422e604ac78971f56a3f3ed3b253c96e0f986c67e44fdc916bdb3defce12095ac839841670dd4157e4aebd7e33dcf3546b06630b1
+  metadata.gz: e2fb79546eb167872017b0995e7c055fd26ffea32272c17403e4a278e4b13f63a6245c2e5bb277061a90c6eddef6ad72e7bb00cead8a9d312058213db8b3a8f2
+  data.tar.gz: 39548d2cfbe17d534a4e03f7060399fbb8e8f06fce93a0190c86fe1360d9064b56bf8cb9c70cfc8e508fe19dd8af8e69107aecfbfab944e323cbd6a935f75471

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.302.3)
+    rest-ftp-daemon (0.304.0)
       api-auth
-      chamber
+      bmc-daemon-lib (~> 0.2)
       double-bag-ftps
       facter
       get_process_mem
@@ -27,6 +27,7 @@ GEM
       i18n (~> 0.7)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
+    addressable (2.4.0)
     api-auth (2.0.0)
     ast (2.3.0)
     astrolabe (1.3.1)
@@ -35,6 +36,8 @@ GEM
       descendants_tracker (~> 0.0.4)
       ice_nine (~> 0.11.0)
       thread_safe (~> 0.3, >= 0.3.1)
+    bmc-daemon-lib (0.2.0)
+      chamber (~> 2.9)
     builder (3.2.2)
     chamber (2.9.0)
       hashie (~> 3.3)
@@ -55,7 +58,7 @@ GEM
     eventmachine (1.2.0.1)
     facter (2.4.6)
       CFPropertyList (~> 2.2.6)
-    ffi (1.9.13)
+    ffi (1.9.14)
     get_process_mem (0.2.1)
     grape (0.16.2)
       activesupport
@@ -73,8 +76,15 @@ GEM
     haml (4.0.7)
       tilt
     hashie (3.4.4)
+    http (2.0.2)
+      addressable (~> 2.3)
+      http-cookie (~> 1.0)
+      http-form_data (~> 1.0.1)
+      http_parser.rb (~> 0.6.0)
     http-cookie (1.0.2)
       domain_name (~> 0.5)
+    http-form_data (1.0.1)
+    http_parser.rb (0.6.0)
     i18n (0.7.0)
     ice_nine (0.11.2)
     json (1.8.3)
@@ -153,6 +163,7 @@ PLATFORMS
 
 DEPENDENCIES
   bundler (~> 1.6)
+  http
   pry
   rake
   rest-ftp-daemon!

data/README.md

@@ -253,7 +253,7 @@ transfer:
 
 newrelic:
   debug: false
-  # licence: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+  # license: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
   # app_name: "rftpd-bigbusiness-dev"     # app_name used for naming app (used as-is if provided)
   prefix: "rftpd"                         # app prefix to build app_name
   # platform: "bigbusiness"               # app platform to build app_name

data/bin/rest-ftp-daemon

@@ -3,15 +3,15 @@
 # Try to load external libs, helpers and constants
 begin
   require "rubygems"
+  require 'bmc-daemon-lib'
   require "optparse"
   require "thin"
   require "socket"
   require_relative "../lib/rest-ftp-daemon/launcher"
-  require_relative "../lib/shared/conf"
 rescue LoadError => e
   raise "EXITING: some basic libs were not found (#{e.message})"
 end
-include Shared
+include BmcDaemonLib
 include RestFtpDaemon
 
 
@@ -119,7 +119,7 @@ puts
 puts "Daemonize       \t #{Conf[:daemonize]}"
 puts "Network port    \t #{Conf[:port]}"
 puts "User:group      \t #{Conf[:user]}:#{Conf[:group]}" if Conf[:user] && Conf[:group]
-puts "Newrelic        \t #{Conf.newrelic_enabled? ? Conf.at(:newrelic, :licence) : 'no'}"
+puts "Newrelic        \t #{Conf.newrelic_enabled? ? Conf.at(:newrelic, :license) : 'no'}"
 puts "PID file        \t #{Conf[:pidfile]}"
 puts
 puts Conf.dump

data/config.ru

@@ -10,7 +10,7 @@ $queue = RestFtpDaemon::JobQueue.new
 # Create global counters
 $counters = RestFtpDaemon::Counters.new
 
-# Initialize workers and conchita subsystem
+# Initialize workers
 $pool = RestFtpDaemon::WorkerPool.new
 
 # Rack authent

data/defaults.yml

@@ -49,13 +49,13 @@ transfer:
 
 newrelic:
   debug: false
-  # licence: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+  # license: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
   # app_name: "rftpd-bigbusiness-dev"     # app_name used for naming app (used as-is if provided)
   prefix: "rftpd"                         # app prefix to build app_name
   # platform: "bigbusiness"               # app platform to build app_name
 
 logs:
-  path:     "/var/log/"
+  path:     "/tmp/"
 
   thin:     "rftpd-environment-thin.log"
   newrelic: "rftpd-environment-newrelic.log"

data/lib/rest-ftp-daemon/api/root.rb

@@ -32,8 +32,8 @@ module RestFtpDaemon
       end
 
       ### CLASS CONFIG
-      helpers Shared::LoggerHelper
-      logger RestFtpDaemon::LoggerPool.instance.get :api
+      helpers BmcDaemonLib::LoggerHelper
+      logger BmcDaemonLib::LoggerPool.instance.get :api
       do_not_route_head!
       do_not_route_options!
       # version 'v1'
@@ -54,8 +54,14 @@ module RestFtpDaemon
         super
 
         # Check that Queue and Pool are available
-        raise RestFtpDaemon::MissingQueue unless defined? $queue
-        raise RestFtpDaemon::MissingPool unless defined? $pool
+        unless $pool.is_a? RestFtpDaemon::WorkerPool
+          log_error "Metrics.sample: invalid WorkerPool"
+          raise RestFtpDaemon::MissingPool
+        end
+        unless $queue.is_a? RestFtpDaemon::JobQueue
+          log_error "Metrics.sample: invalid JobQueue"
+          raise RestFtpDaemon::MissingQueue
+        end
       end
 
 

data/lib/rest-ftp-daemon/api/status.rb

@@ -10,6 +10,16 @@ module RestFtpDaemon
       get "/" do
         status 200
 
+        # Check that Queue and Pool are available
+        unless true
+          log_error "Status API: invalid WorkerPool"
+          return status 501
+        end
+        unless $queue.is_a? RestFtpDaemon::JobQueue
+          log_error "Status API: invalid JobQueue"
+          return status 501
+        end
+
         # Get counters
         counters = $counters.stats.dup
 

data/lib/rest-ftp-daemon/constants.rb

@@ -1,6 +1,3 @@
-# Misc constants
-
-
 # Configuration defaults
 DEFAULT_POOL            = "default"
 DEFAULT_SFTP_TIMEOUT    = 600   # 10mn
@@ -19,7 +16,6 @@ JOB_UPDATE_INTERVAL     = 1
 # Constants: logger
 LOG_ROTATION             = "daily"
 LOG_FORMAT_PROGNAME     = "%d\t%s"
-
 LOG_HEADER_TIME          = "%Y-%m-%d %H:%M:%S"
 LOG_HEADER_FORMAT        = "%s \t%d\t%-8s %-10s "
 LOG_MESSAGE_TRIM         = 200
@@ -27,6 +23,7 @@ LOG_MESSAGE_TEXT         = "%s%s"
 LOG_MESSAGE_ARRAY        = "%s     - %s"
 LOG_MESSAGE_HASH         = "%s     * %-20s %s"
 
+
 # Constants: logger app-specific prefix
 LOG_PREFIX_WID           = 8
 LOG_PREFIX_JID           = JOB_IDENT_LEN + 4

data/lib/rest-ftp-daemon/counters.rb

@@ -1,7 +1,9 @@
-module RestFtpDaemon
+# Queue that stores all the Jobs waiting to be processed or fully processed
 
-  # Queue that stores all the Jobs waiting to be processed or fully processed
+module RestFtpDaemon
   class Counters
+
+    # Class options
     attr_reader :stats
 
     def initialize

data/lib/rest-ftp-daemon/job.rb

@@ -1,14 +1,16 @@
+# Reprensents work to be done along with parameters to process it
 require "securerandom"
 
 module RestFtpDaemon
-
-  # Reprensents work to be done along with parameters to process it
   class Job
-    include Shared::LoggerHelper
-    attr_reader :logger
+    include BmcDaemonLib::LoggerHelper
+    include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
 
+    # Class constants
     FIELDS = [:source, :target, :label, :priority, :pool, :notify, :overwrite, :mkdir, :tempfile]
 
+    # Class options
+    attr_reader :logger
     attr_accessor :wid
 
     attr_reader :id
@@ -51,7 +53,7 @@ module RestFtpDaemon
       @pools        = Conf[:pools] || {}
 
       # Logger
-      @logger = RestFtpDaemon::LoggerPool.instance.get :transfer
+      @logger = BmcDaemonLib::LoggerPool.instance.get :transfer
 
       # Protect with a mutex
       @mutex = Mutex.new
@@ -659,13 +661,10 @@ module RestFtpDaemon
     end
 
     # NewRelic instrumentation
-    if Conf.newrelic_enabled?
-      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
-      add_transaction_tracer :prepare,        category: :task
-      add_transaction_tracer :run,            category: :task
-      add_transaction_tracer :client_notify,  category: :task
-      add_transaction_tracer :initialize,     category: :task
-    end
+    add_transaction_tracer :prepare,        category: :task
+    add_transaction_tracer :run,            category: :task
+    add_transaction_tracer :client_notify,  category: :task
+    add_transaction_tracer :initialize,     category: :task
 
   end
 end

data/lib/rest-ftp-daemon/job_queue.rb

@@ -2,7 +2,10 @@ module RestFtpDaemon
 
   # Queue that stores all the Jobs waiting to be processed or fully processed
   class JobQueue
-    include Shared::LoggerHelper
+    include BmcDaemonLib::LoggerHelper
+    include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+
+    # Class options
     attr_reader :logger
     attr_reader :jobs
 
@@ -20,7 +23,7 @@ module RestFtpDaemon
       @mutex = Mutex.new
 
       # Logger
-      @logger = RestFtpDaemon::LoggerPool.instance.get :queue
+      @logger = BmcDaemonLib::LoggerPool.instance.get :queue
 
       # Identifiers generator
       @last_id = 0
@@ -232,17 +235,12 @@ module RestFtpDaemon
       "#{@prefix}.#{id}"
     end
 
-  private
-
     # NewRelic instrumentation
-    if Conf.newrelic_enabled?
-      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
-      add_transaction_tracer :push,                 category: :task
-      add_transaction_tracer :pop,                  category: :task
-      add_transaction_tracer :expire,               category: :task
-      add_transaction_tracer :rate_by,              category: :task
-      add_transaction_tracer :jobs_by_status,       category: :task
-    end
+    add_transaction_tracer :push,                 category: :task
+    add_transaction_tracer :pop,                  category: :task
+    add_transaction_tracer :expire,               category: :task
+    add_transaction_tracer :rate_by,              category: :task
+    add_transaction_tracer :jobs_by_status,       category: :task
 
   end
 end

data/lib/rest-ftp-daemon/launcher.rb

@@ -1,5 +1,6 @@
 module RestFtpDaemon
   class Launcher
+    # Class constants
     LAUNCHER_PORT_TIMEOUT       = 3
     LAUNCHER_PORT_LOCALHOST     = "127.0.0.1"
 

data/lib/rest-ftp-daemon/metrics.rb

@@ -2,6 +2,10 @@ module RestFtpDaemon
   class Metrics
 
     def self.sample
+      # Check validity of globals
+      return log_error "Metrics.sample: invalid WorkerPool" unless $pool.is_a? RestFtpDaemon::WorkerPool
+      return log_error "Metrics.sample: invalid JobQueue"  unless $queue.is_a? RestFtpDaemon::JobQueue
+
       # Prepare external deps
       mem = GetProcessMem.new
 
@@ -27,6 +31,12 @@ module RestFtpDaemon
         # Init
         counts = {}
 
+        # Check validity of globals
+        unless $pool.is_a? RestFtpDaemon::WorkerPool
+          log_error "Metrics.workers_count_by_status: invalid WorkerPool"
+          return counts
+        end
+
         $pool.worker_variables.group_by do |wid, vars|
           vars[:status]
         end.each do |status, workers|

data/lib/rest-ftp-daemon/notification.rb

@@ -1,13 +1,13 @@
 require 'api_auth'
 require 'rest_client'
 
+# Handle a notification POST using a dedicated thread
 module RestFtpDaemon
-
-  # Handles a notification POST using a dedicated thread
   class Notification
-    include Shared::LoggerHelper
-    attr_reader :logger
+    include BmcDaemonLib::LoggerHelper
 
+    # Class options
+    attr_reader :logger
     attr_accessor :job_id
     attr_accessor :signal
     attr_accessor :error
@@ -26,7 +26,7 @@ module RestFtpDaemon
       @jid = nil
 
       # Logger
-      @logger = RestFtpDaemon::LoggerPool.instance.get :notify
+      @logger = BmcDaemonLib::LoggerPool.instance.get :notify
 
       # Handle the notification
       log_info "initialized [#{@url}]"

data/lib/rest-ftp-daemon/paginate.rb

@@ -1,6 +1,7 @@
 module RestFtpDaemon
   class Paginate
 
+    # Class options
     attr_writer :filter
     attr_accessor :all
 

data/lib/rest-ftp-daemon/path.rb

@@ -1,5 +1,7 @@
 module RestFtpDaemon
   class Path
+
+    # Class options
     attr_accessor :name
     attr_accessor :dir
 

data/lib/rest-ftp-daemon/remote.rb

@@ -1,8 +1,9 @@
+# Handles transfers for Job class
 module RestFtpDaemon
-
-  # Handles transfers for Job class
   class Remote
-    include Shared::LoggerHelper
+    include BmcDaemonLib::LoggerHelper
+
+    # Class options
     attr_reader :logger
     attr_reader :log_prefix
 
@@ -12,7 +13,7 @@ module RestFtpDaemon
 
       # Logger
       @log_prefix = log_prefix || {}
-      @logger = RestFtpDaemon::LoggerPool.instance.get :transfer
+      @logger = BmcDaemonLib::LoggerPool.instance.get :transfer
 
       # Extract URL parts
       @url = url

data/lib/rest-ftp-daemon/remote_ftp.rb

@@ -1,9 +1,11 @@
 require "net/ftp"
 require "double_bag_ftps"
 
+# Handle FTP and FTPeS transfers for Remote class
 module RestFtpDaemon
-  # Handles FTP and FTPeS transfers for Remote class
   class RemoteFTP < Remote
+
+    # Class options
     attr_reader :ftp
 
     def initialize url, log_prefix, options = {}

data/lib/rest-ftp-daemon/remote_sftp.rb

@@ -1,8 +1,10 @@
 require "net/sftp"
 
+# Handle sFTP transfers for Remote class
 module RestFtpDaemon
-  # Handles sFTP transfers for Remote class
   class RemoteSFTP < Remote
+
+    # Class options
     attr_reader :sftp
 
     def initialize url, log_prefix, options = {}
@@ -19,7 +21,7 @@ module RestFtpDaemon
       log_debug "RemoteSFTP.connect [#{@url.user}]@[#{@url.host}]:[#{@url.port}]"
 
       # Debug level
-      verbosity =  @debug ? Logger::INFO : false
+      verbosity =  @debug ? Logger::DEBUG : false
 
       # Connect remote server
       @sftp = Net::SFTP.start(@url.host.to_s, @url.user.to_s,

data/lib/rest-ftp-daemon/views/dashboard_workers.haml

@@ -1,5 +1,8 @@
 -# coding: utf-8
--  variables = $pool.worker_variables
+
+- unless $pool.is_a? RestFtpDaemon::WorkerPool
+  - log_error "Dashboard: invalid WorkerPool"
+- variables = $pool.worker_variables
 
 %h2 Workers
 

data/lib/rest-ftp-daemon/worker_pool.rb

@@ -2,13 +2,16 @@ module RestFtpDaemon
 
   # Handles a pool of Worker objects
   class WorkerPool
-    include Shared::LoggerHelper
+    include BmcDaemonLib::LoggerHelper
+    include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+
+    # Class options
     attr_reader :logger
     attr_reader :wid
 
     def initialize
       # Logger
-      @logger = RestFtpDaemon::LoggerPool.instance.get :workers
+      @logger = BmcDaemonLib::LoggerPool.instance.get :workers
 
       # Prepare status hash and vars
       @statuses = {}
@@ -99,11 +102,7 @@ module RestFtpDaemon
     end
 
     # NewRelic instrumentation
-    if Conf.newrelic_enabled?
-      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
-      # add_transaction_tracer :create_conchita_thread,     category: :task
-      add_transaction_tracer :create_thread,       category: :task
-    end
+    add_transaction_tracer :create_thread, category: :task
 
   end
 end

data/lib/rest-ftp-daemon/workers/conchita.rb

@@ -1,7 +1,7 @@
-module RestFtpDaemon
+# Worker used to clean up the queue deleting expired jobs
 
-  # Worker used to clean up the queue deleting expired jobs
-  class ConchitaWorker < Shared::WorkerBase
+module RestFtpDaemon
+  class ConchitaWorker < Worker
 
   protected
 
@@ -31,10 +31,6 @@ module RestFtpDaemon
 
       # Force garbage collector
       GC.start if @config["garbage_collector"]
-
-    rescue StandardError => e
-      log_error "EXCEPTION: #{e.inspect}"
-      sleep 1
     end
 
   private
@@ -43,5 +39,10 @@ module RestFtpDaemon
       @config["clean_#{status}"] || 0
     end
 
+    # NewRelic instrumentation
+    add_transaction_tracer :worker_init,       category: :task
+    add_transaction_tracer :worker_after,      category: :task
+    add_transaction_tracer :worker_process,    category: :task
+
   end
 end

data/lib/rest-ftp-daemon/workers/reporter.rb

@@ -1,7 +1,7 @@
-module RestFtpDaemon
+# Worker used to report metrics to various services
 
-  # Worker used to report metrics to various services
-  class ReporterWorker < Shared::WorkerBase
+module RestFtpDaemon
+  class ReporterWorker < Worker
 
   protected
 
@@ -9,6 +9,9 @@ module RestFtpDaemon
       # Load corker conf
       config_section :reporter
 
+      # Other configuration options
+      @report_newrelic = Conf.newrelic_enabled?
+
       # Check that everything is OK
       return "invalid timer" unless @config[:timer].to_i > 0
       return false
@@ -26,21 +29,21 @@ module RestFtpDaemon
 
       # Report metrics
       do_metrics
-
-    rescue StandardError => e
-      log_error "EXCEPTION: #{e.inspect}"
-      sleep 1
     end
 
   private
 
     def do_metrics
+      # What metrics to report?
+      report_newrelic = Conf.newrelic_enabled?
+
       # Get common metrics and dump them to logs
+      log_debug "begin metrics sample"
       metrics = Metrics.sample
-      log_info "collected metrics", metrics
+      log_info "collected metrics (newrelic: #{@report_newrelic})", metrics
 
       # Transpose metrics to NewRelic metrics
-      report_newrelic(metrics) if Conf.newrelic_enabled?
+      report_newrelic(metrics) if @report_newrelic
     end
 
     def report_newrelic metrics
@@ -54,10 +57,19 @@ module RestFtpDaemon
       end
 
       # Don't dump metrics unless we're debugging
-      metrics_newrelic = nil unless @config[:debug]
-      newrelic_app_name = ENV["NEW_RELIC_APP_NAME"]
-      log_debug "reported metrics to NewRelic [#{newrelic_app_name}]", metrics_newrelic
+      msg_newrelic = "reported metrics to NewRelic [#{ENV['NEW_RELIC_APP_NAME']}]"
+      if @config[:debug]
+        log_debug msg_newrelic, metrics_newrelic
+      else
+        log_info msg_newrelic
+      end
+
     end
 
+    # NewRelic instrumentation
+    add_transaction_tracer :worker_init,       category: :task
+    add_transaction_tracer :worker_after,      category: :task
+    add_transaction_tracer :worker_process,    category: :task
+
   end
 end

data/lib/rest-ftp-daemon/workers/transfer.rb

@@ -1,7 +1,7 @@
-module RestFtpDaemon
+# Worker used to process Jobs
 
-  # Worker used to process Jobs
-  class TransferWorker < Shared::WorkerBase
+module RestFtpDaemon
+  class TransferWorker < Worker
 
   protected
 
@@ -66,10 +66,6 @@ module RestFtpDaemon
         # Now, requeue this job
         $queue.requeue job
       end
-
-    rescue StandardError => ex
-      log_error "WORKER UNHANDLED EXCEPTION: #{ex.message}", ex.backtrace
-      worker_status WORKER_STATUS_CRASHED
     end
 
     def work_on_job job
@@ -104,5 +100,10 @@ module RestFtpDaemon
       job.oops_after_crash ex unless job.nil?
     end
 
+    # NewRelic instrumentation
+    add_transaction_tracer :worker_init,       category: :task
+    add_transaction_tracer :worker_after,      category: :task
+    add_transaction_tracer :worker_process,    category: :task
+
   end
 end

data/lib/rest-ftp-daemon/workers/worker.rb

@@ -0,0 +1,12 @@
+# Instrumented WorkerBase
+
+module RestFtpDaemon
+  class Worker < BmcDaemonLib::WorkerBase
+    include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+
+    # NewRelic instrumentation
+    add_transaction_tracer :worker_init,       category: :task
+    add_transaction_tracer :worker_after,      category: :task
+    add_transaction_tracer :worker_process,    category: :task
+  end
+end