rest-ftp-daemon 0.247.1 → 0.250.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 09ad0a98b4d033900852d99282cf2e8e259a25df
-  data.tar.gz: 9e10478bbf7e263fcbbeeb1720d5d9ea54a9518a
+  metadata.gz: 56ef0db0f52451ad8dd48acea697310637a460a7
+  data.tar.gz: 8c1c45f0077e055a48c6d80bb1d9f60892019d2e
 SHA512:
-  metadata.gz: 8282173e27456be64a164bc83d58f157dd6351b227121d4f4a53ae1e59bc36e875e52d9404d078a04e14697e1bc320eae23c28679e0706c969dd23f5d6b16b64
-  data.tar.gz: cda917f1dfd9e26260aee4679f708399dc9088e7e36fb4c873dfc3ff1c7c9a742d78df5893b95fe2641fbfef6fdab4d9583a745b8c2eec8954707b222177c599
+  metadata.gz: d8120b207886ab8a2667b68761e539a23cc547d04880844d0898ac6560a2edeea313033dca26de45b6af51f0715a97a3e7ad82e626b3915f2255d530ab0dadee
+  data.tar.gz: 43e8dff779837f56751a1accb9ddf69fe122cc31af4ec97c83ce556028b27199164bb9bc9b7aebe708b1a108e060a3fbb8a8752aae33d21f90eb137ab33d0378

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.247.1)
+    rest-ftp-daemon (0.250.0)
       double-bag-ftps
       facter
       get_process_mem
@@ -13,7 +13,7 @@ PATH
       newrelic_rpm
       settingslogic
       sys-cpu
-      thin (~> 1.6)
+      thin (~> 1.6.4)
 
 GEM
   remote: http://rubygems.org/
@@ -80,8 +80,8 @@ GEM
     ice_nine (0.11.2)
     json (1.8.3)
     method_source (0.8.2)
-    minitest (5.8.4)
-    multi_json (1.12.0)
+    minitest (5.9.0)
+    multi_json (1.12.1)
     multi_xml (0.5.5)
     mustermann19 (0.4.3)
       enumerable-lazy
@@ -120,7 +120,7 @@ GEM
       powerpack (~> 0.1)
       rainbow (>= 1.99.1, < 3.0)
       ruby-progressbar (~> 1.4)
-    ruby-progressbar (1.8.0)
+    ruby-progressbar (1.8.1)
     settingslogic (2.0.9)
     slop (3.6.0)
     sys-cpu (0.7.2)
@@ -130,7 +130,7 @@ GEM
       eventmachine (~> 1.0, >= 1.0.4)
       rack (~> 1.0)
     thread_safe (0.3.5)
-    tilt (2.0.2)
+    tilt (2.0.4)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
     unf (0.1.4)

data/README.md

@@ -214,7 +214,7 @@ TODO for this document
 * Document /status
 * Document /routes
 * Document mkdir and overwrite options
-* Document counters
+* Document stats
 
 
 

data/config.ru

@@ -6,6 +6,9 @@ require "rest-ftp-daemon"
 # Create global queue
 $queue = RestFtpDaemon::JobQueue.new
 
+# Create global stats
+$stats = RestFtpDaemon::Stats.new
+
 # Initialize workers and conchita subsystem
 $pool = RestFtpDaemon::WorkerPool.new
 

data/lib/rest-ftp-daemon.rb

@@ -10,6 +10,8 @@ require "net/http"
 require "thread"
 require "singleton"
 require "newrelic_rpm"
+require "grape"
+require "grape-entity"
 
 # Project's libs
 require_relative "rest-ftp-daemon/constants"
@@ -23,6 +25,7 @@ require_relative "rest-ftp-daemon/logger"
 require_relative "rest-ftp-daemon/paginate"
 require_relative "rest-ftp-daemon/uri"
 require_relative "rest-ftp-daemon/job_queue"
+require_relative "rest-ftp-daemon/stats"
 require_relative "rest-ftp-daemon/worker"
 require_relative "rest-ftp-daemon/worker_conchita"
 require_relative "rest-ftp-daemon/worker_job"
@@ -38,6 +41,9 @@ require_relative "rest-ftp-daemon/remote_sftp"
 require_relative "rest-ftp-daemon/api/job_presenter"
 require_relative "rest-ftp-daemon/api/jobs"
 require_relative "rest-ftp-daemon/api/dashboard"
+require_relative "rest-ftp-daemon/api/status"
+require_relative "rest-ftp-daemon/api/config"
+require_relative "rest-ftp-daemon/api/debug"
 require_relative "rest-ftp-daemon/api/root"
 
 # Haml monkey-patching

data/lib/rest-ftp-daemon/api/config.rb

@@ -0,0 +1,25 @@
+module RestFtpDaemon
+  module API
+    class Config < Grape::API
+
+      desc "Show daemon config"
+      get "/" do
+        status 200
+        return Helpers.get_censored_config
+      end
+
+      desc "Reload daemon config"
+      post "/reload" do
+        if Settings.at(:debug, :allow_reload)==true
+          Settings.reload!
+          status 200
+          return Helpers.get_censored_config
+        else
+          status 403
+          return "Config reload not permitted"
+        end
+      end
+
+    end
+  end
+end

data/lib/rest-ftp-daemon/api/dashboard.rb

@@ -1,21 +1,16 @@
-require "grape"
 require "haml"
 require "sys/cpu"
+require "get_process_mem"
 require "facter"
 
 module RestFtpDaemon
   module API
 
-    # Offers an HTML dashboard through the Grape API (hum...)
     class Dashbaord < Grape::API
 
       ### HELPERS
 
       helpers do
-        def logger
-          Root.logger
-        end
-
         def render name, values={}
           template = File.read("#{APP_LIBS}/views/#{name}.haml")
 
@@ -65,12 +60,6 @@ module RestFtpDaemon
       end
 
 
-      ### Common request logging
-      before do
-        log_info "HTTP #{request.request_method} #{request.fullpath}", params
-      end
-
-
       ### DASHBOARD
       desc "Show a global dashboard"
       get "/" do

data/lib/rest-ftp-daemon/api/debug.rb

@@ -0,0 +1,37 @@
+module RestFtpDaemon
+  module API
+    class Debug < Grape::API
+
+      desc "Show app routes, params encodings"
+      get "/" do
+        # Encodings
+        encodings = {}
+        jobs = $queue.jobs
+
+        jobs.each do |job|
+          # here = out[job.id] = {}
+          me = encodings[job.id] = {}
+
+          me[:error] = job.error.encoding.to_s unless job.error.nil?
+          me[:status] = job.status.encoding.to_s unless job.status.nil?
+
+          Job::FIELDS.each do |name|
+            value = job.send(name)
+            me[name] = value.encoding.to_s if value.is_a? String
+          end
+
+          job.infos.each do |name, value|
+            me["infos_#{name}"] = value.encoding.to_s if value.is_a? String
+          end
+        end
+
+       # Build response
+       return  {
+          routes: RestFtpDaemon::API::Root.routes,
+          encodings: encodings,
+          }
+      end
+
+    end
+  end
+end

data/lib/rest-ftp-daemon/api/job_presenter.rb

@@ -1,5 +1,3 @@
-require "grape-entity"
-
 module RestFtpDaemon
   module API
     module Entities

data/lib/rest-ftp-daemon/api/jobs.rb

@@ -1,26 +1,7 @@
-require "grape"
-
 module RestFtpDaemon
   module API
     class Jobs < Grape::API
 
-      ### HELPERS
-
-      helpers do
-        def logger
-          Root.logger
-        end
-      end
-
-
-      ### Common request logging
-      before do
-        log_info "HTTP #{request.request_method} #{request.fullpath}", params
-      end
-
-
-      ### READ ONE JOB
-
       desc "Read job with ID"
       params do
         requires :id, type: String, desc: "ID of the Job to read"
@@ -47,9 +28,6 @@ module RestFtpDaemon
         end
       end
 
-
-      ### READ ALL JOBS
-
       desc "List all Jobs"
       get "/" do
         begin
@@ -67,9 +45,6 @@ module RestFtpDaemon
         end
       end
 
-
-      ### CREATE A JOB
-
       desc "Create a new job"
       params do
         requires :source, type: String, desc: "Source file pattern"
@@ -102,7 +77,7 @@ module RestFtpDaemon
           $queue.push job
 
           # Increment a counter
-          $queue.counter_inc :jobs_received
+          $stats.increment :jobs, :received
 
         rescue JSON::ParserError => exception
           log_error "JSON::ParserError: #{exception.message}"

data/lib/rest-ftp-daemon/api/root.rb

@@ -1,23 +1,50 @@
-require "grape"
-require "get_process_mem"
-
 module RestFtpDaemon
   module API
     class Root < Grape::API
 
+      ### LOGGING & HELPERS
+
+      helpers do
+        def logger
+          Root.logger
+        end
+
+        def log_request
+          if env.nil?
+            puts "HTTP_ENV_IS_NIL: #{env.inspect}"
+            return
+          end
+
+          request_method = env['REQUEST_METHOD']
+          request_path   = env['REQUEST_PATH']
+          request_uri    = env['REQUEST_URI']
+          log_info "HTTP #{request_method} #{request_uri}", params
+        end
+      end
+
+      before do
+        log_request
+      end
+
+
       ### CLASS CONFIG
 
       helpers RestFtpDaemon::LoggerHelper
       logger RestFtpDaemon::LoggerPool.instance.get :api
-
       do_not_route_head!
       do_not_route_options!
-
+      # version 'v1'
       format :json
       content_type :json, 'application/json; charset=utf-8'
 
+
+      ### MOUNTPOINTS
+
+      mount RestFtpDaemon::API::Status => MOUNT_STATUS
       mount RestFtpDaemon::API::Jobs => MOUNT_JOBS
       mount RestFtpDaemon::API::Dashbaord => MOUNT_BOARD
+      mount RestFtpDaemon::API::Config => MOUNT_CONFIG
+      mount RestFtpDaemon::API::Debug => MOUNT_DEBUG
 
 
       ### INITIALIZATION
@@ -32,114 +59,12 @@ module RestFtpDaemon
       end
 
 
-      ### HELPERS
-
-      helpers do
-        def logger
-          Root.logger
-        end
-      end
-
-
-      ### Common request logging
-
-      before do
-        log_info "HTTP #{request.request_method} #{request.fullpath}", params
-      end
-
-
       ### ROOT URL ACCESS
 
       get "/" do
         redirect Helpers.dashboard_filter_url()
       end
 
-
-      ### SHOW ROUTES
-
-      desc "Show application routes"
-      get "/routes" do
-        status 200
-        return RestFtpDaemon::API::Root.routes
-      end
-
-
-      ### SHOW STATUS
-
-      desc "Show daemon status"
-      get "/status" do
-        mem = GetProcessMem.new
-        status 200
-        return  {
-          hostname: `hostname`.to_s.chomp,
-          version: APP_VER,
-          started: APP_STARTED,
-          uptime: (Time.now - APP_STARTED).round(1),
-          counters: $queue.counters,
-          memory_bytes: mem.bytes.to_i,
-          memory_mb: mem.mb.round(0),
-          status: $queue.counts_by_status,
-          workers: $pool.worker_variables,
-          jobs_count: $queue.jobs_count,
-          jobs_queued: $queue.queued_ids,
-          }
-      end
-
-
-      ### SHOW CONFIG
-
-      desc "Show daemon config"
-      get "/config" do
-        status 200
-        return Helpers.get_censored_config
-      end
-
-
-      desc "List all Jobs params encodings"
-      get "/encodings" do
-        # Get jobs to display
-        encodings = {}
-        jobs = $queue.jobs
-
-        jobs.each do |job|
-          # here = out[job.id] = {}
-          me = encodings[job.id] = {}
-
-          me[:error] = job.error.encoding.to_s unless job.error.nil?
-          me[:status] = job.status.encoding.to_s unless job.status.nil?
-
-          Job::FIELDS.each do |name|
-            value = job.send(name)
-            me[name] = value.encoding.to_s if value.is_a? String
-          end
-
-          job.infos.each do |name, value|
-            me["infos_#{name}"] = value.encoding.to_s if value.is_a? String
-          end
-
-          # # Computed fields
-          # expose :age
-          # expose :exectime
-        end
-
-        encodings
-      end
-
-
-      ### RELOAD CONFIG
-
-      desc "Reload daemon config"
-      post "/config/reload" do
-        if Settings.at(:debug, :allow_reload)==true
-          Settings.reload!
-          status 200
-          return Helpers.get_censored_config
-        else
-          status 403
-          return "Config reload not permitted"
-        end
-      end
-
     end
   end
 end

data/lib/rest-ftp-daemon/api/status.rb

@@ -0,0 +1,41 @@
+require "get_process_mem"
+
+module RestFtpDaemon
+  module API
+    class Status < Grape::API
+
+      desc "Show daemon status"
+      get "/" do
+        mem = GetProcessMem.new
+        status 200
+        return  {
+          hostname: `hostname`.to_s.chomp,
+          version: APP_VER,
+
+          started: APP_STARTED,
+          uptime: (Time.now - APP_STARTED).round(1),
+
+          memory_bytes: mem.bytes.to_i,
+          memory_mb: mem.mb.round(0),
+
+          stats: $stats.stats,
+
+          jobs_count: $queue.jobs_count,
+          status: $queue.jobs_count_by_status,
+
+          # jobs_finished: $queue.jobs_finished,
+          #rate_by_pool1: $queue.rate_by_pool,
+          rate_by_pool: $queue.rate_by(:pool),
+          rate_by_targethost: $queue.rate_by(:targethost),
+          jobs_by_status: $queue.jobs_by_status,
+
+          #rates_per_host: $queue.rates_per_host,
+
+          workers: $pool.worker_variables,
+          #kpis: $queue.queued_ids,
+          }
+      end
+
+    end
+  end
+end

data/lib/rest-ftp-daemon/constants.rb

@@ -1,7 +1,7 @@
 # Terrific constants
 APP_NAME = "rest-ftp-daemon"
 APP_NICK = "rftpd"
-APP_VER = "0.247.1"
+APP_VER = "0.250.0"
 
 # Provide default config file information
 APP_LIB = File.expand_path(File.dirname(__FILE__))
@@ -92,6 +92,9 @@ BIND_PORT_LOCALHOST     = "127.0.0.1"
 ENV_PRODUCTION           = "production"
 MOUNT_JOBS               = "/jobs"
 MOUNT_BOARD              = "/board"
+MOUNT_STATUS             = "/status"
+MOUNT_DEBUG              = "/debug"
+MOUNT_CONFIG             = "/config"
 
 
 # Notifications

data/lib/rest-ftp-daemon/job.rb

@@ -246,6 +246,10 @@ module RestFtpDaemon
       (Time.now - @queued_at).round(2)
     end
 
+    def targethost
+      get(:target_host)
+    end
+
     def json_target
       utf8 @target_method
     end
@@ -314,6 +318,7 @@ module RestFtpDaemon
       raise RestFtpDaemon::JobMissingAttribute unless @target
       target_uri = expand_url @target
       set_info :target_uri, target_uri.to_s
+      set_info :target_host, target_uri.host
       @target_path = Path.new target_uri.path, true
 
       #puts "@target_path: #{@target_path.inspect}"
@@ -485,9 +490,9 @@ module RestFtpDaemon
       set_status JOB_STATUS_DISCONNECTING
       @finished_at = Time.now
 
-      # Update counters
-      $queue.counter_inc :jobs_finished
-      $queue.counter_add :transferred, @transfer_total
+      # Update stats
+      $stats.increment :jobs, :finished
+      $stats.add :global, :transferred, @transfer_total
     end
 
     def remote_push source, target
@@ -652,8 +657,8 @@ module RestFtpDaemon
       end
 
       # Increment counter for this error
-      $queue.counter_inc "err_#{error}"
-      $queue.counter_inc :jobs_failed
+      $stats.increment :errors, error
+      $stats.increment :jobs, :failed
 
       # Prepare notification if signal given
       return unless event

data/lib/rest-ftp-daemon/job_queue.rb

@@ -35,10 +35,6 @@ module RestFtpDaemon
       @last_id = 0
       @prefix = Helpers.identifier JOB_IDENT_LEN
       log_info "JobQueue initialized (prefix: #{@prefix})"
-
-      # Mutex for counters
-      @counters = {}
-      @mutex_counters = Mutex.new
     end
 
     def generate_id
@@ -48,63 +44,58 @@ module RestFtpDaemon
       prefixed_id @last_id
     end
 
-    def counter_add name, value
-      @mutex_counters.synchronize do
-        @counters[name] ||= 0
-        @counters[name] += value
-      end
+      def jobs_queued
+      @queues
+      #@queues.map { |status, jobs| jobs.size }
     end
 
-    def counter_inc name
-      counter_add name, 1
-    end
+    # Statistics on average rates
+    def rate_by method_name
+      # Init
+      rates = {}
+      return unless Job.new(0, {}).respond_to? method_name
 
-    def counter_get name
-      @mutex_counters.synchronize do
-        @counters[name]
+      # Group jobs by method_name
+      jobs_grouped = @jobs.group_by do |job|
+        job.send(method_name)
       end
-    end
 
-    def counters
-      @mutex_counters.synchronize do
-        @counters
+      # Inside each group, sum up rates for interesting statuses
+      jobs_grouped.map do |group, jobs|
+        # Store their sum
+        rates[group] = rates_by_status (jobs)
       end
-    end
 
-    def jobs_queued
-      @queues
+      # Return the rate
+      rates
     end
 
-    def jobs_with_status status
-      # No status filter: return all execept queued
-      if status.empty?
-        @jobs.reject { |job| job.status == JOB_STATUS_QUEUED }
-
-      # Status filtering: only those jobs
-      else
-        @jobs.select { |job| job.status.to_s == status.to_s }
-
-      end
+    # Queue infos
+    def jobs_count
+      @jobs.length
     end
 
-    def counts_by_status
+    def jobs_by_status
       statuses = {}
       @jobs.group_by { |job| job.status }.map { |status, jobs| statuses[status] = jobs.size }
       statuses
     end
+    alias jobs_count_by_status jobs_by_status
 
-    def jobs_count
-      @jobs.length
+    def jobs_ids
+      @jobs.collect(&:id)
     end
 
-    def queued_ids
-      @queues.collect{|pool, jobs| jobs.collect(&:id)}
+    def empty?
+      @queue.empty?
     end
 
-    def jobs_ids
-      @jobs.collect(&:id)
+    def num_waiting
+      @waiting.size
     end
 
+
+    # Queue access
     def find_by_id id, prefixed = false
       # Build a prefixed id if expected
       id = prefixed_id(id) if prefixed
@@ -115,7 +106,6 @@ module RestFtpDaemon
       @jobs.find { |item| item.id == id }
     end
 
-
     def push job
       # Check that item responds to "priorty" method
       raise "JobQueue.push: job should respond to priority method" unless job.respond_to? :priority
@@ -174,42 +164,52 @@ module RestFtpDaemon
     alias shift pop
     alias deq pop
 
-    def empty?
-      @queue.empty?
-    end
-
     def clear
       @queue.clear
     end
 
-    def num_waiting
-      @waiting.size
+    # Jobs acess and searching
+    def jobs_with_status status
+      # No status filter: return all execept queued
+      if status.empty?
+        @jobs.reject { |job| job.status == JOB_STATUS_QUEUED }
+
+      # Status filtering: only those jobs
+      else
+        @jobs.select { |job| job.status.to_s == status.to_s }
+
+      end
     end
 
+    # Jobs cleanup
     def expire status, maxage, verbose = false
 # FIXME: clean both @jobs and @queue
       # Init
       return if status.nil? || maxage <= 0
 
-      # Compute oldest possible birthday
-      before = Time.now - maxage.to_i
-
-      # Verbose output ?
-      log_info "JobQueue.expire \t[#{status}] \tbefore \t[#{before}]" if verbose
+      # Compute oldest limit
+      time_limit = Time.now - maxage.to_i
+      log_info "JobQueue.expire limit [#{time_limit}] status [#{status}]" if verbose
 
       @mutex.synchronize do
         # Delete jobs from the queue when they match status and age limits
         @jobs.delete_if do |job|
+          # log_info "testing job [#{job.id}] updated_at [#{job.updated_at}]"
 
-          # Skip if wrong status, updated_at invalid, or too young
-          next unless job.status == status.to_sym
+          # Skip if wrong status, updated_at invalid, or updated since time_limit
+          next unless job.status == status
           next if job.updated_at.nil?
-          next if job.updated_at > before
+          next if job.updated_at >= time_limit
 
           # Ok, we have to clean it up ..
-          log_info "expire [#{status}] [#{maxage}] > [#{job.id}] [#{job.updated_at}]"
-          log_info "#{LOG_INDENT}unqueued" if @queue.delete(job)
+          log_info "expire [#{status}]: job [#{job.id}] updated_at [#{job.updated_at}]"
 
+          # From any queues, remove it
+          @queues.each do |pool, jobs|
+            log_info "#{LOG_INDENT}unqueued from [#{pool}]" if jobs.delete(job)
+          end
+
+          # Remember we have to delete the original job !
           true
         end
       end
@@ -222,11 +222,32 @@ module RestFtpDaemon
       "#{@prefix}.#{id}"
     end
 
+    def rates_by_status jobs
+      rates = {}
+
+      # Sub-group by status
+      jobs.group_by(&:status).each do |status, jobset|
+
+        # Extract bitrate values
+        bitrates = jobset.collect do |job|
+          job.get(:transfer_bitrate)
+        end
+
+        # Store their sum
+        rates[status] = bitrates.reject(&:nil?).sum
+        # rates["#{status}_ids"] = jobset.collect(&:id).join(', ')
+      end
+
+      # Return rates
+      rates
+    end
+
     if Settings.newrelic_enabled?
-      add_transaction_tracer :push,             category: :task
-      add_transaction_tracer :pop,              category: :task
-      add_transaction_tracer :expire,           category: :task
-      add_transaction_tracer :counts_by_status, category: :task
+      add_transaction_tracer :push,                 category: :task
+      add_transaction_tracer :pop,                  category: :task
+      add_transaction_tracer :expire,               category: :task
+      add_transaction_tracer :rate_by, category: :task
+      add_transaction_tracer :jobs_count_by_status, category: :task
     end
 
   end

data/lib/rest-ftp-daemon/notification.rb

@@ -75,8 +75,9 @@ module RestFtpDaemon
       # Post notification, handle server response / multi-lines
       log_info "sending #{data}"
       response = http.post uri.path, data, headers
-      response_lines = response.body.lines
 
+      # Log reponse body
+      response_lines = response.body.lines
       if response_lines.size > 1
         human_size = Helpers.format_bytes(response.body.bytesize, "B")
         log_info "received [#{response.code}] #{human_size} (#{response_lines.size} lines)", response_lines
@@ -85,8 +86,14 @@ module RestFtpDaemon
       end
 
       # Handle exceptions
+      rescue Net::OpenTimeout => ex
+        log_error "Net::OpenTimeout: #{ex.inspect}"
+
+      rescue SocketError => ex
+        log_error "SocketError: #{ex.inspect}"
+
       rescue StandardError => ex
-        log_error "NOTIFICATION EXCEPTION: #{ex.inspect}"
+        log_error "UNHANDLED EXCEPTION: #{ex.inspect}"
     end
 
     def log_context

data/lib/rest-ftp-daemon/stats.rb

@@ -0,0 +1,42 @@
+module RestFtpDaemon
+
+  # Queue that stores all the Jobs waiting to be processed or fully processed
+  class Stats
+    attr_reader :stats
+
+    if Settings.newrelic_enabled?
+      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+    end
+
+    def initialize
+      @stats = {}
+      @mutex_stats = Mutex.new
+    end
+
+    def set group, name, value
+      @mutex_stats.synchronize do
+        @stats[group] ||= {}
+        @stats[group][name] = value
+      end
+    end
+
+    def get group, name
+      @mutex_stats.synchronize do
+        @stats[group][name] if @stats[group]
+      end
+    end
+
+    def add group, name, value
+      @mutex_stats.synchronize do
+        @stats[group] ||= {}
+        @stats[group][name] ||= 0
+        @stats[group][name] += value
+      end
+    end
+
+    def increment group, name
+      add group, name, 1
+    end
+
+  end
+end

data/lib/rest-ftp-daemon/views/dashboard.haml

@@ -29,9 +29,10 @@
 
         #box-workers.col-md-3
           = render :dashboard_workers
+          = render :dashboard_stats
 
-        #box-counters.col-md-3
-          = render :dashboard_counters
+        #box-rates.col-md-3
+          = render :dashboard_rates
 
 
     .footer

data/lib/rest-ftp-daemon/views/dashboard_footer.haml

@@ -1,5 +1,4 @@
 -# coding: utf-8
-- trans = $queue.counter_get :transferred
 - info_procs = (Facter.value :processorcount).to_i
 
 

data/lib/rest-ftp-daemon/views/dashboard_header.haml

@@ -1,9 +1,9 @@
 -# coding: utf-8
-- trans = $queue.counter_get :transferred
 - info_procs = (Facter.value :processorcount).to_i
 - info_load = Sys::CPU.load_avg.first.to_f
 - info_norm = info_procs.zero? ? "N/A" : (100 * info_load / info_procs).round(1)
-- info_processed = $queue.counter_get(:jobs_processed)
+- info_trans = $stats.get :global, :transferred
+- info_processed = $stats.get(:global, :processed)
 - mem = GetProcessMem.new
 
 
@@ -35,6 +35,6 @@
 
   .btn-group.btn-group-sm
     .btn.btn-default.btn-success Transferred
-    .btn.btn-default= Helpers.format_bytes(trans, "B", 1)
+    .btn.btn-default= Helpers.format_bytes(info_trans, "B", 1)
 
 

data/lib/rest-ftp-daemon/views/dashboard_jobs.haml

@@ -1,5 +1,5 @@
 -# coding: utf-8
-- counts_by_status = $queue.counts_by_status
+- jobs_count_by_status = $queue.jobs_count_by_status
 - counts_all = $queue.jobs_count
 - jobs = @paginate.subset
 
@@ -9,7 +9,7 @@
     ALL (#{counts_all})
 
 .btn-group.btn-group-xs.filters
-  - counts_by_status.each do |status, count|
+  - jobs_count_by_status.each do |status, count|
     - klass = (status.to_s == @filter) ? "btn-info" : ""
     %a.btn.btn-default{href: Helpers.dashboard_filter_url(status), class: klass}
       #{status} (#{count})
@@ -50,10 +50,10 @@
             %tbody.jobs
               = render :dashboard_table, {jobs: jobs}
 
-              %thead
-                %tr
-                  %td{colspan: 14}
-                    %br
+            %thead
+              %tr
+                %td{colspan: 14}
+                  %br
 
         - unless jobs.empty?
           %tbody.jobs

data/lib/rest-ftp-daemon/views/dashboard_rates.haml

@@ -0,0 +1,36 @@
+-# coding: utf-8
+
+- groups = {pool: "pool", targethost: "target host"}
+
+
+%h2 Transfer rates
+
+%table.table.table-striped.table-hover.table-condensed
+
+  - groups.each do |group_by, group_title|
+    - rates_by_status = $queue.rate_by(group_by)
+
+    %thead
+      %tr
+        %th= group_title
+        %th.text-right=JOB_STATUS_FINISHED
+        %th.text-right=JOB_STATUS_UPLOADING
+
+    %tbody
+      - rates_by_status.each do |group, rates|
+        - next if group.nil?
+
+        %tr.info
+          %td= group
+          %td.text-right
+            = Helpers.format_bytes(rates[JOB_STATUS_FINISHED], "bps")
+          %td.text-right
+            = Helpers.format_bytes(rates[JOB_STATUS_UPLOADING], "bps")
+          -# %td= rates.inspect
+
+    %thead
+      %tr
+        %td{colspan: 3}
+          %br
+
+    -# JOB_STATUS_UPLOADING

data/lib/rest-ftp-daemon/views/dashboard_stats.haml

@@ -0,0 +1,21 @@
+-# coding: utf-8
+%h2 Stats
+
+%table.table.table-striped.table-hover.table-condensed
+
+  %thead
+    %tr
+      %th group
+      %th name
+      %th.text-right value
+
+  %tbody
+
+    - $stats.stats.each do |group, values|
+
+      - values.each do |name, value|
+        %tr
+          %td= group
+          %td= name
+          %td.text-right= value
+

data/lib/rest-ftp-daemon/views/dashboard_table.haml

@@ -77,14 +77,6 @@
         .label.label-default.flag.worker-label= job.priority
 
     %td
-
       .label.flag.worker-label{class: Helpers.job_runs_style(runs)}= runs
 
-    -#%td
-      - unless job.priority.nil?
-        .label.label-default.flag.worker-label= job.priority
-
-      - unless job.wid.nil?
-        .label.label-warning.flag.worker-label= job.wid
-
 

data/lib/rest-ftp-daemon/views/dashboard_tokens.haml

@@ -3,11 +3,13 @@
 
 %table.table.table-striped.table-hover.table-condensed
 
-  %tr
-    %th token
-    %th value
-
-  - tokens.each do |token, value|
+  %thead
     %tr
-      %td= Helpers.tokenize token
-      %td= Helpers.hide_credentials_from_url value
+      %th token
+      %th value
+
+  %tbody
+    - tokens.each do |token, value|
+      %tr
+        %td= Helpers.tokenize token
+        %td= Helpers.hide_credentials_from_url value

data/lib/rest-ftp-daemon/views/dashboard_workers.haml

@@ -3,33 +3,35 @@
 
 %table.table.table-striped.table-hover.table-condensed
 
-  %tr
-    %th ID
-    %th pool
-    %th status
-    %th job
-    %th.text-right seen
+  %thead
+    %tr
+      %th pool
+      %th worker
+      %th status
+      %th job
+      %th.text-right seen
 
-  - @worker_variables.each do |vars|
-    - wid = vars[:wid]
-    - status = vars[:status]
-    - alive = $pool.worker_alive? wid
-    - trclass = WORKER_STYLES[status]
+  %tbody
+    - @worker_variables.each do |vars|
+      - wid = vars[:wid]
+      - status = vars[:status]
+      - alive = $pool.worker_alive? wid
+      - trclass = WORKER_STYLES[status]
 
-    - unless alive
-      - trclass = "danger"
-      - status = "DEAD"
+      - unless alive
+        - trclass = "danger"
+        - status = "DEAD"
 
 
-    %tr{class: trclass.to_s}
-      %td= wid
-      %td= vars[:pool]
-      %td= status
-      %td= vars[:jid]
-      %td.text-right
+      %tr{class: trclass.to_s}
+        %td= vars[:pool]
+        %td= wid
+        %td= status
+        %td= vars[:jid]
+        %td.text-right
 
-        - if vars[:updated_at].is_a? Time
-          - no_news_for = (Time.now - vars[:updated_at]).round(0)
-          = Helpers.formatted_duration no_news_for
-        - else
-          = "?"
+          - if vars[:updated_at].is_a? Time
+            - no_news_for = (Time.now - vars[:updated_at]).round(0)
+            = Helpers.formatted_duration no_news_for
+          - else
+            = "?"

data/lib/rest-ftp-daemon/worker_job.rb

@@ -86,7 +86,7 @@ module RestFtpDaemon
       worker_status WORKER_STATUS_FINISHED, job
 
       # Increment total processed jobs count
-      $queue.counter_inc :jobs_processed
+      $stats.increment :global, :processed
 
     rescue RestFtpDaemon::JobTimeout => ex
       log_error "JOB TIMED OUT", ex.backtrace

data/rest-ftp-daemon.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rubocop", "~> 0.32.0"
   spec.add_development_dependency "pry"
 
-  spec.add_runtime_dependency "thin", "~> 1.6"
+  spec.add_runtime_dependency "thin", "~> 1.6.4"
   spec.add_runtime_dependency "grape"
   spec.add_runtime_dependency "grape-entity"
   spec.add_runtime_dependency "settingslogic"

data/spec/rest-ftp-daemon/features/{routes_spec.rb → debug_spec.rb}

@@ -1,10 +1,10 @@
 require "spec_helper"
 
-describe "Routes", feature: true do
+describe "Debug", feature: true do
 
-  let!(:response) { get "/routes" }
+  let!(:response) { get MOUNT_DEBUG }
 
-  describe "GET /routes" do
+  describe "GET #{MOUNT_DEBUG}" do
 
     it "responds successfully" do
       expect(response.status).to eq 200

data/spec/rest-ftp-daemon/features/status_spec.rb

@@ -2,9 +2,9 @@ require "spec_helper"
 
 describe "Status", feature: true do
 
-  let!(:response) { get "/status" }
+  let!(:response) { get MOUNT_STATUS }
 
-  describe "GET /status" do
+  describe "GET #{MOUNT_DEBUG}" do
 
     it "responds successfully" do
       expect(response.status).to eq 200

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: 0.247.1
+  version: 0.250.0
 platform: ruby
 authors:
 - Bruno MEDICI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-09 00:00:00.000000000 Z
+date: 2016-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -100,14 +100,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: 1.6.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: 1.6.4
 - !ruby/object:Gem::Dependency
   name: grape
   requirement: !ruby/object:Gem::Requirement
@@ -282,10 +282,13 @@ files:
 - bin/rest-ftp-daemon
 - config.ru
 - lib/rest-ftp-daemon.rb
+- lib/rest-ftp-daemon/api/config.rb
 - lib/rest-ftp-daemon/api/dashboard.rb
+- lib/rest-ftp-daemon/api/debug.rb
 - lib/rest-ftp-daemon/api/job_presenter.rb
 - lib/rest-ftp-daemon/api/jobs.rb
 - lib/rest-ftp-daemon/api/root.rb
+- lib/rest-ftp-daemon/api/status.rb
 - lib/rest-ftp-daemon/array.rb
 - lib/rest-ftp-daemon/constants.rb
 - lib/rest-ftp-daemon/exceptions.rb
@@ -305,12 +308,14 @@ files:
 - lib/rest-ftp-daemon/settings.rb
 - lib/rest-ftp-daemon/static/css/bootstrap.css
 - lib/rest-ftp-daemon/static/css/main.css
+- lib/rest-ftp-daemon/stats.rb
 - lib/rest-ftp-daemon/uri.rb
 - lib/rest-ftp-daemon/views/dashboard.haml
-- lib/rest-ftp-daemon/views/dashboard_counters.haml
 - lib/rest-ftp-daemon/views/dashboard_footer.haml
 - lib/rest-ftp-daemon/views/dashboard_header.haml
 - lib/rest-ftp-daemon/views/dashboard_jobs.haml
+- lib/rest-ftp-daemon/views/dashboard_rates.haml
+- lib/rest-ftp-daemon/views/dashboard_stats.haml
 - lib/rest-ftp-daemon/views/dashboard_table.haml
 - lib/rest-ftp-daemon/views/dashboard_tokens.haml
 - lib/rest-ftp-daemon/views/dashboard_workers.haml
@@ -321,8 +326,8 @@ files:
 - rest-ftp-daemon.gemspec
 - rest-ftp-daemon.yml.sample
 - spec/rest-ftp-daemon/features/dashboard_spec.rb
+- spec/rest-ftp-daemon/features/debug_spec.rb
 - spec/rest-ftp-daemon/features/jobs_spec.rb
-- spec/rest-ftp-daemon/features/routes_spec.rb
 - spec/rest-ftp-daemon/features/status_spec.rb
 - spec/spec_helper.rb
 - spec/support/config.yml

data/lib/rest-ftp-daemon/views/dashboard_counters.haml

@@ -1,15 +0,0 @@
--# coding: utf-8
-%h2 Counters
-
-%table.table.table-striped.table-hover.table-condensed
-
-  %tr
-    %th name
-    %th.text-right value
-
-  - $queue.counters.each do |name, value|
-
-    %tr
-      %td= name
-      %td.text-right= value
-