rest-ftp-daemon 0.214.0 → 0.220.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a453a7be5c87798246b4603a6787365b5829f58f
-  data.tar.gz: 3583043f7aa342bc2992fc5a8b0599a19acfe693
+  metadata.gz: 519c3872919419beaa88f1dc46d3f41fb798effe
+  data.tar.gz: 245b623aeab51fd699aeb7f5842a517ea19b1188
 SHA512:
-  metadata.gz: 94e64e9196678fa87b03d0ada12c1f7b4064e5c88d1996731b060a9703c901d858e2f0aab525f0b3ad5a12e15aad076e51d9b6e9fcd2292cad24788c78606368
-  data.tar.gz: 3933f23e1bbe6c039ef4f94c69e0bf682504514e8718a9df14d9c9c4f7466357dbf2a190170a8bc143975b5c1ff6543b52c15d4627f86f4afb360f11c87dd3ad
+  metadata.gz: a5e8da4e45ed4c753f3ce6c978b8b8a070ae73b368db26410743f0c281f631365528cf54362134d8607ee9aa9cb401d0653c73942f43fa121dc43a2d711d99e5
+  data.tar.gz: b595327455aeb85e1f60969f09de446f37dbdfc0ef213d34ee10a93f460b53007104d5dc254b1c37f6df37eebd5fdedd1aa54cf790927a716806802136ca83a6

data/.gitignore

@@ -3,5 +3,6 @@ pkg
 .DS_Store
 *.log
 tmp/
+log/
 DOC/
 rest-ftp-daemon.yml

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.214.0)
+    rest-ftp-daemon (0.220.0)
       double-bag-ftps
       facter
       get_process_mem

data/bin/rest-ftp-daemon

@@ -114,7 +114,7 @@ puts "Host         \t #{Settings.host}"
 puts "Namespace    \t #{Settings.namespace}"
 puts "Network port \t #{Settings.port}"
 puts "User:group   \t #{Settings.user}:#{Settings.group}" if (Settings.user || Settings.group)
-puts "Newrelic     \t #{Settings.newrelic_enabled? ? Settings.newrelic.inspect : "no"}"
+puts "Newrelic     \t #{Settings.newrelic_enabled? ? Settings.at(:debug, :newrelic) : "no"}"
 # puts Settings.dump
 puts
 puts "--- Thin ARGV"

data/config.ru

@@ -7,7 +7,7 @@ require 'rest-ftp-daemon'
 $queue = RestFtpDaemon::JobQueue.new
 
 # Initialize workers and conchita subsystem
-$pool = RestFtpDaemon::WorkerPool.new(Settings.workers || APP_WORKERS)
+$pool = RestFtpDaemon::WorkerPool.new
 
 # Rack reloader
 unless Settings.namespace == "production"

data/lib/rest-ftp-daemon.rb

@@ -16,6 +16,8 @@ require 'net/http'
 require 'double_bag_ftps'
 require 'thread'
 require 'securerandom'
+require 'singleton'
+require 'logger'
 
 require 'newrelic_rpm'
 require 'get_process_mem'
@@ -26,20 +28,22 @@ require 'rest-ftp-daemon/constants'
 require 'rest-ftp-daemon/settings'
 require 'rest-ftp-daemon/exceptions'
 require 'rest-ftp-daemon/helpers'
+require 'rest-ftp-daemon/logger_helper'
+require 'rest-ftp-daemon/logger_pool'
+require 'rest-ftp-daemon/logger'
 require 'rest-ftp-daemon/paginate'
 require 'rest-ftp-daemon/uri'
 require 'rest-ftp-daemon/job_queue'
 # require 'rest-ftp-daemon/worker'
-require 'rest-ftp-daemon/conchita'
+require 'rest-ftp-daemon/worker'
+require 'rest-ftp-daemon/worker_conchita'
+require 'rest-ftp-daemon/worker_job'
 require 'rest-ftp-daemon/worker_pool'
-require 'rest-ftp-daemon/logger'
-require 'rest-ftp-daemon/logger_pool'
 require 'rest-ftp-daemon/job'
 require 'rest-ftp-daemon/notification'
 require 'rest-ftp-daemon/api/root'
 require 'rest-ftp-daemon/api/jobs'
-require 'rest-ftp-daemon/api/debug'
-require 'rest-ftp-daemon/api/routes'
 require 'rest-ftp-daemon/api/dashboard'
 require 'rest-ftp-daemon/api/status'
+require 'rest-ftp-daemon/api/debug'
 require 'rest-ftp-daemon/api/job_presenter'

data/lib/rest-ftp-daemon/api/dashboard.rb

@@ -13,7 +13,7 @@ module RestFtpDaemon
 
       # Server global status
       get '/' do
-        info "GET /"
+        log_info "GET /"
 
         # Initialize Facter
         Facter.loadfacts

data/lib/rest-ftp-daemon/api/debug.rb

@@ -5,7 +5,7 @@ module RestFtpDaemon
 ####### GET /debug
 
       get '/raise' do
-        info "GET /debug"
+        log_info "GET /debug"
         begin
           raise RestFtpDaemon::DummyException
         rescue RestFtpDaemon::RestFtpDaemonException => exception
@@ -20,24 +20,11 @@ module RestFtpDaemon
         end
       end
 
-      # get '/memsize' do
-      #   info "GET /memsize"
-      #   ObjectSpace.each_object do |e|
-      #     #puts
-      #     print ObjectSpace.memsize_of(e)
-      #     print "\t"
-      #     print e.class.to_s
-      #     print "\t"
-      #     puts e.inspect[0..80]
-      #     # puts ({
-      #     #   klass: e.class,
-      #     #   size: ObjectSpace.memsize_of(e),
-      #     # }.inspect)
-      #     # puts
-      #   end
-      #   status 200
-      #   []
-      # end
+      get '/routes' do
+        info "GET /routes"
+        status 200
+        return RestFtpDaemon::API::Root::routes
+      end
 
     end
   end

data/lib/rest-ftp-daemon/api/jobs.rb

@@ -9,7 +9,7 @@ module RestFtpDaemon
         requires :id, type: String, desc: 'ID of the Job to read', regexp: /[^\/]+/
       end
       get '/jobs/*id' do
-        info "GET /jobs/#{params[:id]}"
+        log_info "GET /jobs/#{params[:id]}"
 
         begin
           # Get job to display
@@ -18,15 +18,15 @@ module RestFtpDaemon
           raise RestFtpDaemon::JobNotFound if job.nil?
 
         rescue RestFtpDaemon::JobNotFound => exception
-          info "EXCEPTION: JobNotFound: #{exception.message}"
+          log_error "JobNotFound: #{exception.message}"
           status 404
           api_error exception
         rescue RestFtpDaemonException => exception
-          info "EXCEPTION: RestFtpDaemonException: #{exception.message}"
+          log_error "RestFtpDaemonException: #{exception.message}"
           status 500
           api_error exception
         rescue Exception => exception
-          info "EXCEPTION: Exception: #{exception.message}"
+          log_error "Exception: #{exception.message}"
           status 501
           api_error exception
         else
@@ -42,7 +42,7 @@ module RestFtpDaemon
       desc "List all Jobs"
 
       get '/jobs/' do
-        info "GET /jobs"
+        log_info "GET /jobs"
 
         begin
           # Detect QS filters
@@ -53,11 +53,11 @@ module RestFtpDaemon
           jobs = $queue.jobs
 
         rescue RestFtpDaemonException => exception
-          info "EXCEPTION: RestFtpDaemonException: #{exception.message}"
+          log_error "RestFtpDaemonException: #{exception.message}"
           status 501
           api_error exception
         rescue Exception => exception
-          info "EXCEPTION: Exception: #{exception.message}"
+          log_error "Exception: #{exception.message}"
           status 501
           api_error exception
         else
@@ -88,7 +88,7 @@ module RestFtpDaemon
       end
 
       post '/jobs/' do
-        info "POST /jobs", params.collect {|name, value| "#{name}: #{value.inspect}"}
+        log_info "POST /jobs", params
 
         begin
 
@@ -103,15 +103,15 @@ module RestFtpDaemon
           $queue.counter_inc :jobs_received
 
         rescue JSON::ParserError => exception
-          info "EXCEPTION: JSON::ParserError: #{exception.message}"
+          log_error "JSON::ParserError: #{exception.message}"
           status 406
           api_error exception
         rescue RestFtpDaemonException => exception
-          info "EXCEPTION: RestFtpDaemonException: #{exception.message}"
+          log_error "RestFtpDaemonException: #{exception.message}"
           status 412
           api_error exception
         rescue Exception => exception
-          info "EXCEPTION: #{exception.message}"
+          log_error "Exception: #{exception.message}"
           status 501
           api_error exception
         else

data/lib/rest-ftp-daemon/api/root.rb

@@ -5,7 +5,8 @@ module RestFtpDaemon
 
 ####### CLASS CONFIG
 
-      # logger RestFtpDaemon::Logger.new(:api, "API")
+      helpers RestFtpDaemon::LoggerHelper
+
       logger RestFtpDaemon::LoggerPool.instance.get :api
 
       do_not_route_head!
@@ -33,12 +34,14 @@ module RestFtpDaemon
 
       helpers do
 
-        def info message, lines = []
-          Root.logger.info_with_id message,
-            lines: lines,
-            origin: self.class.to_s
+        def logger
+          Root.logger
         end
 
+        # def log_context
+        #   {}
+        # end
+
         def api_error exception
           {
           :error => exception.message,

data/lib/rest-ftp-daemon/api/status.rb

@@ -7,7 +7,7 @@ module RestFtpDaemon
 
       # Server global status
       get '/status' do
-        info "GET /status"
+        log_info "GET /status"
         mem = GetProcessMem.new
 
         status 200

data/lib/rest-ftp-daemon/constants.rb

@@ -1,7 +1,7 @@
 # Terrific constants
 APP_NAME = "rest-ftp-daemon"
 APP_NICK = "rftpd"
-APP_VER = "0.214.0"
+APP_VER = "0.220.0"
 
 
 # Jobs and workers
@@ -22,7 +22,11 @@ LOG_COL_JID             = JOB_IDENT_LEN+3+2
 LOG_COL_ID              = 6
 LOG_TRIM_LINE           = 80
 LOG_DUMPS               = File.dirname(__FILE__) + '/../../log/'
-
+LOG_ROTATION            = 'daily'
+LOG_FORMAT_TIME         = "%Y-%m-%d %H:%M:%S"
+LOG_FORMAT_PREFIX       = "%s %s\t%-#{LOG_PIPE_LEN.to_i}s\t"
+LOG_FORMAT_MESSAGE      = "%#{-LOG_COL_WID.to_i}s\t%#{-LOG_COL_JID.to_i}s\t%#{-LOG_COL_ID.to_i}s"
+LOG_NEWLINE             = "\n"
 
 # Notifications
 NOTIFY_PREFIX           = "rftpd"
@@ -45,7 +49,7 @@ WORKER_STYLES = {
   :done                 => :success,
   :dead                 => :danger
   }
-PAGINATE_MAX            = 30
+PAGINATE_MAX            = 40
 
 
 # Configuration defaults

data/lib/rest-ftp-daemon/exceptions.rb

@@ -6,6 +6,7 @@ module RestFtpDaemon
 
   class MissingQueue              < RestFtpDaemonException; end
   class MissingPool               < RestFtpDaemonException; end
+  class InvalidWorkerNumber       < RestFtpDaemonException; end
 
   class JobException              < RestFtpDaemonException; end
   class JobTimeout                < RestFtpDaemonException; end

data/lib/rest-ftp-daemon/job.rb

@@ -1,5 +1,7 @@
 module RestFtpDaemon
   class Job
+    include LoggerHelper
+    attr_reader :logger
 
     if Settings.newrelic_enabled?
       include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
@@ -65,14 +67,14 @@ module RestFtpDaemon
       @updated_at = Time.now
 
       # Send first notification
-      info "Job.initialize notify[queued] notify_after_sec[#{@notify_after_sec}] update_every_kb[#{@update_every_kb}]"
+      log_info "Job.initialize notify[queued] notify_after_sec[#{@notify_after_sec}] update_every_kb[#{@update_every_kb}]"
       client_notify :queued
     end
 
     def process
       # Update job's status
       @error = nil
-      info "Job.process starting"
+      log_info "Job.process starting"
 
       # Prepare job
       begin
@@ -110,7 +112,7 @@ module RestFtpDaemon
       else
         # Prepare done !
         newstatus :prepared
-        info "Job.process notify[started]"
+        log_info "Job.process notify[started]"
         client_notify :started
       end
 
@@ -183,7 +185,7 @@ module RestFtpDaemon
       else
         # All done !
         newstatus JOB_STATUS_FINISHED
-        info "Job.process notify[ended]"
+        log_info "Job.process notify[ended]"
         client_notify :ended
       end
 
@@ -318,7 +320,7 @@ module RestFtpDaemon
 
       # Log detected names
       source_names = source_matches.map{ |s| Helpers.extract_filename s }
-      info "Job.transfer sources #{source_names.inspect}"
+      log_info "Job.transfer sources #{source_names.inspect}"
 
       # Asserts and counters
       raise RestFtpDaemon::JobSourceNotFound if source_matches.empty?
@@ -371,15 +373,11 @@ module RestFtpDaemon
 
   private
 
-    def info message, lines = []
-      return if @logger.nil?
-
-      # Forward to logger
-      @logger.info_with_id message,
-        wid: @wid,
-        jid: @id,
-        lines: lines,
-        origin: self.class.to_s
+    def log_context
+      {
+      wid: @wid,
+      jid: @id
+      }
     end
 
     def newstatus name
@@ -406,7 +404,7 @@ module RestFtpDaemon
       raise RestFtpDaemon::JobAssertionFailed, "ftp_init/1" if @target_method.nil?
       raise RestFtpDaemon::JobAssertionFailed, "ftp_init/2" if @target_url.nil?
 
-      info "Job.ftp_init target_method [#{@target_method}]"
+      log_info "Job.ftp_init target_method [#{@target_method}]"
       case @target_method
       when :ftp
         @ftp = Net::FTP.new
@@ -415,7 +413,7 @@ module RestFtpDaemon
         @ftp.ssl_context = DoubleBagFTPS.create_ssl_context(:verify_mode => OpenSSL::SSL::VERIFY_NONE)
         @ftp.ftps_mode = DoubleBagFTPS::EXPLICIT
       else
-        info "Job.ftp_init unknown scheme [#{@target_url.scheme}]"
+        log_info "Job.ftp_init unknown scheme [#{@target_url.scheme}]"
         railse RestFtpDaemon::JobTargetUnsupported
       end
 
@@ -442,7 +440,7 @@ module RestFtpDaemon
     def ftp_finished
       # Close FTP connexion and free up memory
       @ftp.close
-      info "Job.ftp_finished closed"
+      log_info "Job.ftp_finished closed"
       @ftp = nil
 
       # FTP debug mode ?
@@ -465,7 +463,7 @@ module RestFtpDaemon
 
       # Method assertions
       host = @target_url.host
-      info "Job.ftp_connect [#{host}]"
+      log_info "Job.ftp_connect [#{host}]"
       raise RestFtpDaemon::JobAssertionFailed, "ftp_connect/1" if @ftp.nil?
       raise RestFtpDaemon::JobAssertionFailed, "ftp_connect/2" if @target_url.nil?
 
@@ -481,25 +479,25 @@ module RestFtpDaemon
 
       # use "anonymous" if user is empty
       login = @target_url.user || "anonymous"
-      info "Job.ftp_login [#{login}]"
+      log_info "Job.ftp_login [#{login}]"
 
       @ftp.login login, @target_url.password
     end
 
     def ftp_chdir_or_buildpath path
       # Method assertions
-      info "Job.ftp_chdir [#{path}] mkdir: #{@mkdir}"
+      log_info "Job.ftp_chdir [#{path}] mkdir: #{@mkdir}"
       newstatus :ftp_chdir
       raise RestFtpDaemon::JobAssertionFailed, "ftp_chdir_or_buildpath/1" if path.nil?
 
       # Extract directory from path
       if @mkdir
         # Split dir in parts
-        info "Job.ftp_chdir buildpath [#{path}]"
+        log_info "Job.ftp_chdir buildpath [#{path}]"
         ftp_buildpath path
       else
         # Directly chdir if not mkdir requested
-        info "Job.ftp_chdir chdir [#{path}]"
+        log_info "Job.ftp_chdir chdir [#{path}]"
         @ftp.chdir path
       end
     end
@@ -515,7 +513,7 @@ module RestFtpDaemon
       rescue Net::FTPPermError => exception
         # If not possible because the directory is missing
         parent =  Helpers.extract_parent(path)
-        info "#{pref} chdir failed - parent [#{parent}]"
+        log_info "#{pref} chdir failed - parent [#{parent}]"
 
         # And only if we still have something to "dive up into"
         if parent
@@ -523,18 +521,18 @@ module RestFtpDaemon
           ftp_buildpath parent
 
           # Then finally create this dir and chdir
-          info "#{pref} > now mkdir [#{path}]"
+          log_info "#{pref} > now mkdir [#{path}]"
           @ftp.mkdir path
 
           # And get into it (this chdir is not rescue'd on purpose in order to throw the ex)
-          info "#{pref} > now chdir [#{path}]"
+          log_info "#{pref} > now chdir [#{path}]"
           @ftp.chdir(path)
         end
 
       end
 
       # Now we were able to chdir inside, just tell it
-      info "#{pref} > ftp.pwd [#{@ftp.pwd}]"
+      log_info "#{pref} > ftp.pwd [#{@ftp.pwd}]"
     end
 
     def ftp_presence target_name
@@ -548,7 +546,7 @@ module RestFtpDaemon
 
       # Get file list, sometimes the response can be an empty value
       results = @ftp.list(target_name) rescue nil
-      info "Job.ftp_presence: #{results.inspect}"
+      log_info "Job.ftp_presence: #{results.inspect}"
 
       # Result can be nil or a list of files
       return false if results.nil?
@@ -557,13 +555,13 @@ module RestFtpDaemon
 
     def ftp_transfer source_filename, target_name = nil
       # Method assertions
-      info "Job.ftp_transfer source: #{source_filename}"
+      log_info "Job.ftp_transfer source: #{source_filename}"
       raise RestFtpDaemon::JobAssertionFailed, "ftp_transfer/1" if @ftp.nil?
       raise RestFtpDaemon::JobAssertionFailed, "ftp_transfer/2" if source_filename.nil?
 
       # Use source filename if target path provided none (typically with multiple sources)
       target_name ||= Helpers.extract_filename source_filename
-      info "Job.ftp_transfer target: #{target_name}"
+      log_info "Job.ftp_transfer target: #{target_name}"
       set :source_current, target_name
 
       # Check for target file presence
@@ -572,11 +570,11 @@ module RestFtpDaemon
       if present
         if @overwrite
           # delete it first
-          info "Job.ftp_transfer removing target file"
+          log_info "Job.ftp_transfer removing target file"
           @ftp.delete(target_name)
         else
           # won't overwrite then stop here
-          info "Job.ftp_transfer failed: target file exists"
+          log_info "Job.ftp_transfer failed: target file exists"
           raise RestFtpDaemon::JobTargetFileExists
         end
       end
@@ -585,7 +583,7 @@ module RestFtpDaemon
       target_real = target_name
       if @tempfile
         target_real = "#{target_name}.temp-#{Helpers.identifier(JOB_TEMPFILE_LEN)}"
-        info "Job.ftp_transfer tempfile: #{target_real}"
+        log_info "Job.ftp_transfer tempfile: #{target_real}"
       end
 
       # Start transfer
@@ -606,7 +604,7 @@ module RestFtpDaemon
       # Rename temp file to target_temp
       if @tempfile
         newstatus JOB_STATUS_RENAMING
-        info "Job.ftp_transfer renaming: #{target_name}"
+        log_info "Job.ftp_transfer renaming: #{target_name}"
         @ftp.rename target_real, target_name
       end
 
@@ -615,7 +613,7 @@ module RestFtpDaemon
 
       # Done
       set :source_current, nil
-      info "Job.ftp_transfer finished"
+      log_info "Job.ftp_transfer finished"
     end
 
     def ftp_transfer_block block
@@ -639,7 +637,7 @@ module RestFtpDaemon
       stack << (Helpers.format_bytes @transfer_total, "B")
       stack << (Helpers.format_bytes bitrate0, "bps")
       stack2 = stack.map{ |txt| ("%#{LOG_PIPE_LEN.to_i}s" % txt)}.join("\t")
-      info "Job.ftp_transfer #{stack2}"
+      log_info "Job.ftp_transfer #{stack2}"
 
       # Update time pointer
       @transfer_pointer_at = Time.now
@@ -668,7 +666,7 @@ module RestFtpDaemon
       RestFtpDaemon::Notification.new @notify, payload
 
     rescue Exception => ex
-      info "Job.client_notify EXCEPTION: #{ex.inspect}"
+      log_error "Job.client_notify EXCEPTION: #{ex.inspect}"
      end
 
     def get_bitrate total, last_timestamp
@@ -685,9 +683,9 @@ module RestFtpDaemon
 
       message = "Job.oops event[#{event.to_s}] error[#{error.to_s}] ex[#{exception.class}] #{exception.message}"
       if include_backtrace
-        info message, exception.backtrace
+        log_error message, exception.backtrace
       else
-        info message
+        log_error message
       end
 
       # Close ftp connexion if open
@@ -718,7 +716,10 @@ module RestFtpDaemon
     end
 
     if Settings.newrelic_enabled?
-      add_transaction_tracer :process, :category => :task
+      add_transaction_tracer :prepare, :category => :task
+      add_transaction_tracer :transfer, :category => :task
+      add_transaction_tracer :client_notify, :category => :task
+      add_transaction_tracer :initialize, :category => :task
     end
 
   end