rest-ftp-daemon 0.210.2 → 0.212.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 37f7d8425d04c02fd6bbf0e29e4aa2e68b4a4a88
-  data.tar.gz: 55887f0ff182963af1b010bf29529b73b1665d46
+  metadata.gz: 53e2cba7fe2218d3f261566678feba235343ca4a
+  data.tar.gz: 75c4001a985741740c66165c6b1d73a893d0e43a
 SHA512:
-  metadata.gz: 2d654b0cd05e3acb927cf5c2280c5eb0127d7de59fe6fa69b47eab9e6e0d353fa8f4972054402b28fa4bd67aebffa30fa25ff656edee764a9f4e08f2c9df4a1b
-  data.tar.gz: 2ecd5231f676eb178d8c521a03a53d83a21904a221541371d925d508839b8bf63d5cd019958ba062fd2d97f2fdee4154e7de1fcd5ac7704cf83b5ab02fedbfe9
+  metadata.gz: 695516939d86fe5e962458758dfe2ce3aa34b5c80ca8eec9bd3ccc4f893c303f9fac4489eec4368cc0ab469d53c584654573e938ad6b90a37c42795f3c1d85ce
+  data.tar.gz: 72cdbba9f002f17faa9e6c0d203c089b89e5eb9a57cc32e892287ebe620e4e4c96f1234fc54ca0087e1cfe733ba7da3290c247c93f1f07cf1d1ec269f7ee6361

data/Gemfile.lock

@@ -1,13 +1,15 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.210.2)
+    rest-ftp-daemon (0.212.0)
       double-bag-ftps
       facter
+      get_process_mem
       grape
       grape-entity
       haml
       json
+      newrelic_rpm
       settingslogic
       sys-cpu
       thin (~> 1.6)
@@ -39,6 +41,7 @@ GEM
     facter (2.4.1)
       CFPropertyList (~> 2.2.6)
     ffi (1.9.8)
+    get_process_mem (0.2.0)
     grape (0.11.0)
       activesupport
       builder
@@ -61,6 +64,7 @@ GEM
     minitest (5.5.1)
     multi_json (1.11.0)
     multi_xml (0.5.5)
+    newrelic_rpm (3.11.1.284)
     rack (1.6.0)
     rack-accept (0.4.5)
       rack (>= 0.4)

data/bin/rest-ftp-daemon

@@ -6,9 +6,10 @@ begin
   require "thin"
   require 'optparse'
   require 'socket'
-  require 'timeout'
+  # require 'timeout'
+  require "settingslogic"
 rescue LoadError
-  raise "EXITING: some of basic libs were not found: thin, optparse, socket, timeout"
+  raise "EXITING: some of basic libs were not found: thin, optparse, socket, settingslogic"
 end
 
 
@@ -50,17 +51,26 @@ else
 end
 
 
-# Build configuration file path from options
+# Load config, and merge options from ARGV into settings
 APP_CONF ||= File.expand_path "/etc/#{APP_NAME}.yml"
 abort "EXITING: cannot read configuration file: #{APP_CONF}" unless File.exists? APP_CONF
-
-
-# Load config, and merge options from ARGV into settings
 begin
+  # Import settings
   require File.expand_path("#{app_root}/lib/rest-ftp-daemon/settings")
+
+  # Set defaults
+  Settings.init_defaults
+
+  # Overwrite with commandline options
   Settings.merge!(options)
+
+  # Init NewRelic
+  Settings.init_newrelic
+
 rescue Psych::SyntaxError => e
   abort "EXITING: config file syntax error: #{e.message}"
+rescue Exception => e
+  abort "EXITING: unknow error loading settings  #{e.inspect}"
 end
 
 
@@ -84,30 +94,28 @@ argv << ["--log", Settings.at(:logs, :thin)]
 argv << ["--daemonize"] if [1, true].include? Settings.daemonize
 
 
-# ARGV: logging
+# ARGV: logging, user and group, command
 if (logfile = Settings.at :logs, :thin)
   argv << ["--log", logfile] if logfile
 end
-
-# ARGV: user and group
 if Settings.user && Settings.group
   argv << ["--user", Settings.user]
   argv << ["--group", Settings.group]
 end
-
-
-# ARGV: add command
 argv << command unless command.nil?
 
+
 # Display final configuration
 puts "--- #{APP_NAME} #{APP_VER}"
 puts "Config file  \t #{APP_CONF}"
 puts "PID file     \t #{Settings.pidfile}"
-puts "Daemonize    \t #{Settings.daemonize ? 'YES' : "no"}"
+puts "Daemonize    \t #{Settings.daemonize ? 'YES' : "no (PID: #{Process.pid})"}"
+puts "Host         \t #{Settings.host}"
 puts "Namespace    \t #{Settings.namespace}"
 puts "Network port \t #{Settings.port}"
 puts "User:group   \t #{Settings.user}:#{Settings.group}" if (Settings.user || Settings.group)
-puts Settings.dump
+puts "Newrelic     \t #{Settings.newrelic_enabled? ? Settings.newrelic.inspect : "no"}"
+# puts Settings.dump
 puts
 puts "--- Thin ARGV"
 puts argv.flatten.join(' ')
@@ -139,11 +147,11 @@ rescue SystemExit
   begin
     Process.kill(0, pid)
     puts "Process ##{pid} is running"
-  rescue Errno::EPERM                     # changed uid
-      puts "No permission to query process ##{pid}!";
+  rescue Errno::EPERM                            # changed uid
+    puts "No permission to query process ##{pid}!";
   rescue Errno::ESRCH
-      puts "Process ##{pid} is NOT running.";      # or zombied
+    puts "Process ##{pid} is NOT running.";      # or zombied
   rescue
-      puts "Unable to determine status for ##{pid}: #{$!}"
+    puts "Unable to determine status for ##{pid}: #{$!}"
   end
 end

data/config.ru

@@ -19,6 +19,9 @@ unless Settings.adminpwd.nil?
   end
 end
 
+# NewRelic profiling
+GC::Profiler.enable if Settings.newrelic_enabled?
+
 # Serve static assets
 use Rack::Static, :urls => ["/css", "/images"], :root => "#{APP_LIBS}/static/"
 

data/lib/rest-ftp-daemon/api/debug.rb

@@ -4,7 +4,7 @@ module RestFtpDaemon
 
 ####### GET /debug
 
-      get '/debug' do
+      get '/raise' do
         info "GET /debug"
         begin
           raise RestFtpDaemon::DummyException
@@ -20,6 +20,25 @@ module RestFtpDaemon
         end
       end
 
+      # get '/memsize' do
+      #   info "GET /memsize"
+      #   ObjectSpace.each_object do |e|
+      #     #puts
+      #     print ObjectSpace.memsize_of(e)
+      #     print "\t"
+      #     print e.class.to_s
+      #     print "\t"
+      #     puts e.inspect[0..80]
+      #     # puts ({
+      #     #   klass: e.class,
+      #     #   size: ObjectSpace.memsize_of(e),
+      #     # }.inspect)
+      #     # puts
+      #   end
+      #   status 200
+      #   []
+      # end
+
     end
   end
 end

data/lib/rest-ftp-daemon/api/status.rb

@@ -8,18 +8,22 @@ module RestFtpDaemon
       # Server global status
       get '/status' do
         info "GET /status"
+        mem = GetProcessMem.new
+
         status 200
         return  {
           hostname: `hostname`.chomp,
           version: APP_VER,
-          config: Helpers.get_censored_config,
           started: APP_STARTED,
           uptime: (Time.now - APP_STARTED).round(1),
           counters: $queue.counters,
+          memory_bytes: mem.bytes.to_i,
+          memory_mb: mem.mb.round(0),
           status: $queue.counts_by_status,
           workers: $pool.worker_variables,
           jobs_count: $queue.jobs_count,
-          jobs_queued: $queue.queued_ids
+          jobs_queued: $queue.queued_ids,
+          config: Helpers.get_censored_config,
           #routes: RestFtpDaemon::API::Root::routes,
           }
       end

data/lib/rest-ftp-daemon/constants.rb

@@ -1,6 +1,7 @@
 # Terrific constants
 APP_NAME = "rest-ftp-daemon"
-APP_VER = "0.210.2"
+APP_NICK = "rftpd"
+APP_VER = "0.212.0"
 
 
 # Jobs and workers

data/lib/rest-ftp-daemon/job.rb

@@ -1,6 +1,10 @@
 module RestFtpDaemon
   class Job
 
+    if Settings.newrelic_enabled?
+      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+    end
+
     FIELDS = [:source, :target, :label, :priority, :notify, :overwrite, :mkdir, :tempfile]
 
     attr_accessor :wid
@@ -20,7 +24,6 @@ module RestFtpDaemon
       attr_reader name
     end
 
-
     def initialize job_id, params={}
       # Call super
       # super()
@@ -134,7 +137,7 @@ module RestFtpDaemon
       rescue Errno::ECONNREFUSED => exception
         return oops :ended, exception, :conn_refused
 
-      rescue Timeout::Error, Errno::ETIMEDOUT => exception
+      rescue Timeout::Error, Errno::ETIMEDOUT, Net::ReadTimeout => exception
         return oops :ended, exception, :conn_timed_out
 
       rescue OpenSSL::SSL::SSLError => exception
@@ -144,7 +147,7 @@ module RestFtpDaemon
         return oops :ended, exception, :ftp_perm_error
 
       rescue Net::FTPTempError => exception
-        return oops :ended, exception, :ftp_temp_error
+        return oops :ended, exception, :net_temp_error
 
       rescue Errno::EMFILE => exception
         return oops :ended, exception, :too_many_open_files
@@ -197,6 +200,11 @@ module RestFtpDaemon
       @weight = [@priority.to_i, -@queued_at.to_i]
     end
 
+    def exectime
+      return nil if (@started_at.nil? || @finished_at.nil?)
+      (@finished_at - @started_at).round(2)
+    end
+
     def set_queued
       # Update job status
       newstatus JOB_STATUS_QUEUED
@@ -217,11 +225,6 @@ module RestFtpDaemon
       (Time.now - @queued_at).round(2)
     end
 
-    def exectime
-      return nil if (@started_at.nil? || @finished_at.nil?)
-      (@finished_at - @started_at).round(2)
-    end
-
     def set attribute, value
       @mutex.synchronize do
         @params || {}
@@ -713,5 +716,9 @@ module RestFtpDaemon
       client_notify event, error: error, status: notif_status
     end
 
+    if Settings.newrelic_enabled?
+      add_transaction_tracer :process, :category => :task
+    end
+
   end
 end

data/lib/rest-ftp-daemon/job_queue.rb

@@ -9,6 +9,10 @@ module RestFtpDaemon
     attr_reader :queue
     attr_reader :jobs
 
+    if Settings.newrelic_enabled?
+      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+    end
+
     def initialize
       # Instance variables
       @queue = []
@@ -154,7 +158,7 @@ module RestFtpDaemon
     alias << push
     alias enq push
 
-    def pop(non_block=false)
+    def pop non_block=false
       @mutex.synchronize do
         while true
           if @queue.empty?
@@ -163,10 +167,6 @@ module RestFtpDaemon
             @waiting.push Thread.current
             @mutex.sleep
           else
-            # Refresh queue order
-            # sort_queue!
-
-            # Extract the heaviest item in the queue
             return @queue.pop
           end
         end
@@ -261,5 +261,10 @@ module RestFtpDaemon
         origin: self.class.to_s
     end
 
+    if Settings.newrelic_enabled?
+      add_transaction_tracer :push, :category => :task
+      add_transaction_tracer :pop, :category => :task
+    end
+
   end
 end

data/lib/rest-ftp-daemon/settings.rb

@@ -1,10 +1,3 @@
-# Try to load Settingslogic
-begin
-  require "settingslogic"
-rescue LoadError
-  raise "warning: Settingslogic is needed to provide configuration values to the Gemspec file"
-end
-
 # Configuration class
 class Settings < Settingslogic
   # Read configuration
@@ -27,4 +20,31 @@ class Settings < Settingslogic
     self.to_hash.to_yaml( :Indent => 4, :UseHeader => true, :UseVersion => false )
   end
 
+  def init_defaults
+    Settings['host'] ||= `hostname`.chomp.split('.').first
+  end
+
+  def newrelic_enabled?
+    Settings.newrelic.is_a?(Hash) && Settings.at(:newrelic, :license)
+  end
+
+  def init_newrelic
+    # Skip if not enabled
+    return ENV['NEWRELIC_AGENT_ENABLED'] = 'false' unless Settings.newrelic_enabled?
+
+    # Enable module
+    ENV['NEWRELIC_AGENT_ENABLED'] = 'true'
+    ENV['NEW_RELIC_MONITOR_MODE'] = 'true'
+    #Settings['newrelic']['enabled'] = true
+
+    # License
+    ENV['NEW_RELIC_LICENSE_KEY'] = Settings.at(:newrelic, :license)
+
+    # Appname
+    ENV['NEW_RELIC_APP_NAME'] = Settings.at(:newrelic, :appname) || "#{APP_NICK}-#{Settings.host}-#{APP_ENV}"
+
+    # Logfile
+    ENV['NEW_RELIC_LOG'] = Settings.at(:logs, :newrelic)
+  end
+
 end

data/lib/rest-ftp-daemon/views/dashboard_headers.haml

@@ -2,33 +2,38 @@
 - info_procs = (Facter.value :processorcount).to_i
 - info_load = Sys::CPU.load_avg.first.to_f
 - info_norm = info_procs.zero? ? "N/A" : (100 * info_load / info_procs).round(1)
+- info_processed = $queue.counter_get(:jobs_processed)
+- mem = GetProcessMem.new
 
 
 .btn-group.btn-group-sm
-  .btn.btn-default.btn-success CPUs
-  .btn.btn-default= info_procs
+  .btn.btn-default.btn-success Host
+  .btn.btn-default= Settings.host
 
 .btn-group.btn-group-sm
   .btn.btn-default.btn-success IP
   .btn.btn-default= Facter.value(:ipaddress)
 
 .btn-group.btn-group-sm
-  .btn.btn-default.btn-success Host
-  .btn.btn-default= Settings.host
+  .btn.btn-default.btn-success Cores
+  .btn.btn-default= info_procs
 
 .btn-group.btn-group-sm
-  .btn.btn-default.btn-info Load
+  .btn.btn-default.btn-warning Load
   .btn.btn-default= info_load.round(1)
 
 .btn-group.btn-group-sm
-  .btn.btn-default.btn-info CPU
+  .btn.btn-default.btn-warning CPU
   .btn.btn-default= "#{info_norm} %"
 
 .btn-group.btn-group-sm
-  .btn.btn-default.btn-info Free
-  .btn.btn-default= Facter.value(:memoryfree)
+  .btn.btn-default.btn-warning Mem
+  .btn.btn-default= Helpers.format_bytes(mem.bytes, "B")
+
+.btn-group.btn-group-sm
+  .btn.btn-default.btn-info Processed
+  .btn.btn-default= Helpers.format_bytes(info_processed)
 
 .btn-group.btn-group-sm
   .btn.btn-default.btn-info Transferred
   .btn.btn-default= Helpers.format_bytes(trans, "B")
-

data/lib/rest-ftp-daemon/views/dashboard_jobs.haml

@@ -26,8 +26,8 @@
       %th <=>
       %th target
       %th queued
-      %th{width: 220} status
-      %th{width: 150} error
+      %th{width: 150} status
+      %th{"min-width" => 120} error
       %th.text-right size
       %th.text-right bitrate
       %th info

data/lib/rest-ftp-daemon/views/dashboard_table.haml

@@ -1,13 +1,10 @@
 - jobs.each do |job|
-  - size = job.get :transfer_total
-  - sent = job.get :transfer_sent
   - progress = job.get :progress
   - source_count = job.get(:source_count) || 0
   - source_processed = job.get(:source_processed) || 0
   - source_current = job.get(:source_current)
   - presented = present job, :with => RestFtpDaemon::API::Entities::JobPresenter, hide_params: true
-  - errmsg = job.get :error_message
-  - method = job.get(:target_method)
+  - bitrate = job.get :transfer_bitrate
 
   - trclass = JOB_STYLES[job.status]
 
@@ -25,7 +22,7 @@
       = Helpers.highlight_tokens job.source
 
     %td
-      = Helpers.job_method_label method
+      = Helpers.job_method_label (job.get :target_method)
 
     %td{title: job.get(:target_url)}
       = Helpers.highlight_tokens job.target
@@ -45,28 +42,26 @@
         - unless progress.nil?
           %span.push-progress
             = "#{progress}%"
-
         %br
-        %b
-          %span.push-filename
-            = source_current unless source_current.nil?
+        %span.push-filename
+          %b= source_current unless source_current.nil?
 
     %td
       - unless job.error || job.status == JOB_STATUS_FINISHED
         .progress
           .progress-bar{style:"width: #{progress}%;"}
             =# "#{progress}%" unless progress.nil?
-            = Helpers.format_bytes(sent, "B")
+            = Helpers.format_bytes(job.get(:transfer_sent), "B")
 
       - else
-        .error{title: errmsg}
+        .error{title: job.get(:error_message)}
           = Helpers.text_or_empty(job.error)
 
     %td.nobr.text-right
-      = Helpers.format_bytes(size, "B")
+      = Helpers.format_bytes(job.get(:transfer_total), "B")
 
-    %td.nobr.text-right
-      - if (bitrate = job.get :transfer_bitrate)
+    %td.nobr.text-right{title: "time: #{job.exectime} s"}
+      - if bitrate
         %span.push-bitrate
           = Helpers.format_bytes(bitrate, "bps")
 
@@ -76,3 +71,5 @@
 
       - unless job.wid.nil?
         .label.label-warning.flag.worker-label= job.wid
+
+

data/lib/rest-ftp-daemon/worker_pool.rb

@@ -3,6 +3,10 @@ module RestFtpDaemon
 
     attr_reader :wid
 
+    if Settings.newrelic_enabled?
+      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+    end
+
     def initialize number_threads
       # Logger
       @logger = RestFtpDaemon::LoggerPool.instance.get :workers
@@ -37,7 +41,6 @@ module RestFtpDaemon
       @workers[wid] && @workers[wid].alive?
     end
 
-
   private
 
     def create_worker_threads n
@@ -55,17 +58,22 @@ module RestFtpDaemon
 
     def create_worker_thread wid
       Thread.new wid do
-
         # Set thread context
         Thread.current.thread_variable_set :wid, wid
         Thread.current.thread_variable_set :started_at, Time.now
+        worker_status :starting
 
         # Start working
-        worker_status :starting
         loop do
-          work
+          begin
+            work
+          rescue Exception => ex
+            puts "WORKER UNEXPECTED CRASH: #{ex.message}", lines: ex.backtrace
+            sleep 1
+          end
         end
 
+        # We should never get here
       end
     end
 
@@ -102,7 +110,7 @@ module RestFtpDaemon
       sleep 1
 
     rescue Exception => ex
-      info "[#{job.id}] UNHDNALED EXCEPTION: #{ex.message}", lines: ex.backtrace
+      info "UNHDNALED EXCEPTION: #{ex.message}", lines: ex.backtrace
       worker_status :crashed
       job.oops_after_crash ex unless job.nil?
       sleep 1
@@ -136,5 +144,10 @@ module RestFtpDaemon
       Thread.current.thread_variable_set :updted_at, Time.now
     end
 
+    if Settings.newrelic_enabled?
+      add_transaction_tracer :create_worker_thread, :category => :task
+      add_transaction_tracer :work, :category => :task
+    end
+
   end
 end

data/lib/rest-ftp-daemon.rb

@@ -1,5 +1,6 @@
 # Global libs
 require 'rubygems'
+require "settingslogic"
 require 'json'
 require 'grape'
 require 'grape-entity'
@@ -14,6 +15,8 @@ require 'net/ftp'
 require 'net/http'
 require 'double_bag_ftps'
 
+require 'newrelic_rpm'
+require 'get_process_mem'
 
 # Project's libs
 require 'rest-ftp-daemon/constants'

data/rest-ftp-daemon.gemspec

@@ -1,6 +1,7 @@
 # coding: utf-8
 
 # Libs
+require "settingslogic"
 app_root = File.dirname(__FILE__)
 require File.expand_path("#{app_root}/lib/rest-ftp-daemon/constants")
 
@@ -36,4 +37,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "facter"
   spec.add_runtime_dependency "sys-cpu"
   spec.add_runtime_dependency "timeout"
+  spec.add_runtime_dependency "get_process_mem"
+
+  spec.add_runtime_dependency "newrelic_rpm"
 end

data/rest-ftp-daemon.yml.sample

@@ -15,12 +15,15 @@ defaults: &defaults
     overwrite: false
     timeout: 1800
 
+  # newrelic:
+    # appname: rftpd-testing
+    # license: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 
   conchita:
     timer: 10
-    #clean_failed: 3600
-    #clean_finished: 3600
-    #clean_queued: 86400
+    # clean_failed: 3600
+    # clean_finished: 3600
+    # clean_queued: 86400
 
   debug:
     ftp: false
@@ -32,6 +35,7 @@ defaults: &defaults
     workers: "/var/log/rftpd-environment-work.log"
     jobs:    "/var/log/rftpd-environment-work.log"
     notify:  "/var/log/rftpd-environment-work.log"
+    newrelic:"/var/log/rftpd-environment-newrelic.log"
 
 preprod:
   <<: *defaults

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: 0.210.2
+  version: 0.212.0
 platform: ruby
 authors:
 - Bruno MEDICI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-25 00:00:00.000000000 Z
+date: 2015-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -178,6 +178,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: get_process_mem
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: newrelic_rpm
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This is a pretty simple FTP client daemon, controlled through a RESTful
   API
 email: rest-ftp-daemon@bmconseil.com