rest-ftp-daemon 0.250.0 → 0.250.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 56ef0db0f52451ad8dd48acea697310637a460a7
-  data.tar.gz: 8c1c45f0077e055a48c6d80bb1d9f60892019d2e
+  metadata.gz: 9162a08e5a91acc98b1fd1f8fe6f56fe06546f35
+  data.tar.gz: ae4b76ce6b9dbf8ff240797c5365d453430ca1d4
 SHA512:
-  metadata.gz: d8120b207886ab8a2667b68761e539a23cc547d04880844d0898ac6560a2edeea313033dca26de45b6af51f0715a97a3e7ad82e626b3915f2255d530ab0dadee
-  data.tar.gz: 43e8dff779837f56751a1accb9ddf69fe122cc31af4ec97c83ce556028b27199164bb9bc9b7aebe708b1a108e060a3fbb8a8752aae33d21f90eb137ab33d0378
+  metadata.gz: bc0ec566787db7bbb2dfb9c54941e80dc070a868ab7fa1b643f311acb0c893675b4067ee5645bd557a29327d0fbd9992b20bce88f2ff7b0ecd1e719258e314b4
+  data.tar.gz: 48d4bf8f2576b314a03cf33a3dace71000aaa23efc6c54e99af19541f14b89d593ee9a9508e533e2ed470f4deb3ad822b2c70ce054702fc5a9e562550b9b9133

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.250.0)
+    rest-ftp-daemon (0.250.3)
       double-bag-ftps
       facter
       get_process_mem
@@ -13,7 +13,7 @@ PATH
       newrelic_rpm
       settingslogic
       sys-cpu
-      thin (~> 1.6.4)
+      thin (~> 1.6)
 
 GEM
   remote: http://rubygems.org/

data/README.md

@@ -4,7 +4,7 @@ rest-ftp-daemon
 [![Gem Version](https://badge.fury.io/rb/rest-ftp-daemon.svg)](http://badge.fury.io/rb/rest-ftp-daemon)
 [![Code Climate](https://codeclimate.com/github/bmedici/rest-ftp-daemon/badges/gpa.svg)](https://codeclimate.com/github/bmedici/rest-ftp-daemon)
 [![Test Coverage](https://codeclimate.com/github/bmedici/rest-ftp-daemon/badges/coverage.svg)](https://codeclimate.com/github/bmedici/rest-ftp-daemon/coverage)
-
+[ ![Codeship](https://codeship.com/projects/46492fd0-00a4-0134-6639-66759d382560/status?branch=master)](https://codeship.com/projects/153245)
 
 A pretty simple but configurable and efficient FTP-client daemon, driven
 through a RESTful API. Create transfer jobs by POSTing a simple JSON structure,
@@ -28,6 +28,7 @@ Features
   * static path pointers in configuration to abstract local mounts or remote FTPs (endpoint tokens)
   * local source path and local/remote target path can use patterns to match multiple files (`/dir/file*.jpg`)
   * several file transfer protocols supported: FTPs, FTPes, sFTP
+  * display bitrate to any pool or any FTP destination currently transferring (API and dashboard)
 
 * Job management
   * highly parrallel job processing using dedicated worker threads with their own context

data/config.ru

@@ -6,8 +6,8 @@ require "rest-ftp-daemon"
 # Create global queue
 $queue = RestFtpDaemon::JobQueue.new
 
-# Create global stats
-$stats = RestFtpDaemon::Stats.new
+# Create global counters
+$counters = RestFtpDaemon::Counters.new
 
 # Initialize workers and conchita subsystem
 $pool = RestFtpDaemon::WorkerPool.new

data/lib/rest-ftp-daemon.rb

@@ -25,7 +25,7 @@ require_relative "rest-ftp-daemon/logger"
 require_relative "rest-ftp-daemon/paginate"
 require_relative "rest-ftp-daemon/uri"
 require_relative "rest-ftp-daemon/job_queue"
-require_relative "rest-ftp-daemon/stats"
+require_relative "rest-ftp-daemon/counters"
 require_relative "rest-ftp-daemon/worker"
 require_relative "rest-ftp-daemon/worker_conchita"
 require_relative "rest-ftp-daemon/worker_job"

data/lib/rest-ftp-daemon/api/dashboard.rb

@@ -37,9 +37,6 @@ module RestFtpDaemon
             @jobs_queued = []
           end
 
-          # Get workers status
-          @worker_variables = $pool.worker_variables
-
           # Build paginator
           @paginate = Paginate.new jobs_with_status
           @paginate.filter = filter

data/lib/rest-ftp-daemon/api/job_presenter.rb

@@ -15,7 +15,7 @@ module RestFtpDaemon
         # expose :error
         expose :json_error, as: :error
         expose :json_status, as: :status
-        expose :json_target, as: :target_method
+        #expose :json_target, as: :target_method
 
         expose :queued_at
         expose :updated_at

data/lib/rest-ftp-daemon/api/jobs.rb

@@ -77,7 +77,7 @@ module RestFtpDaemon
           $queue.push job
 
           # Increment a counter
-          $stats.increment :jobs, :received
+          $counters.increment :jobs, :received
 
         rescue JSON::ParserError => exception
           log_error "JSON::ParserError: #{exception.message}"

data/lib/rest-ftp-daemon/api/status.rb

@@ -8,6 +8,20 @@ module RestFtpDaemon
       get "/" do
         mem = GetProcessMem.new
         status 200
+
+        # Get counters
+        counters = $counters.stats
+
+        # Amend counters with legacy attributes
+        if counters[:jobs].is_a? Hash
+          counters[:jobs_finished] = counters[:jobs][:finished] || 0
+          counters[:jobs_failed]   = counters[:jobs][:failed] || 0
+        end
+        if counters[:data].is_a? Hash
+          counters[:transferred] = counters[:data][:transferred] || 0
+        end
+
+        # Generate sutrcture
         return  {
           hostname: `hostname`.to_s.chomp,
           version: APP_VER,
@@ -18,21 +32,15 @@ module RestFtpDaemon
           memory_bytes: mem.bytes.to_i,
           memory_mb: mem.mb.round(0),
 
-          stats: $stats.stats,
-
+          status: $queue.jobs_by_status,
           jobs_count: $queue.jobs_count,
-          status: $queue.jobs_count_by_status,
 
-          # jobs_finished: $queue.jobs_finished,
-          #rate_by_pool1: $queue.rate_by_pool,
+          counters: counters,
+
           rate_by_pool: $queue.rate_by(:pool),
           rate_by_targethost: $queue.rate_by(:targethost),
-          jobs_by_status: $queue.jobs_by_status,
-
-          #rates_per_host: $queue.rates_per_host,
 
           workers: $pool.worker_variables,
-          #kpis: $queue.queued_ids,
           }
       end
 

data/lib/rest-ftp-daemon/constants.rb

@@ -1,7 +1,7 @@
 # Terrific constants
 APP_NAME = "rest-ftp-daemon"
 APP_NICK = "rftpd"
-APP_VER = "0.250.0"
+APP_VER = "0.250.3"
 
 # Provide default config file information
 APP_LIB = File.expand_path(File.dirname(__FILE__))
@@ -57,6 +57,14 @@ JOB_STYLES = {
   JOB_STATUS_RENAMING    => :info,
   }
 
+
+# Jobs statuses
+JOB_METHOD_FTP           = "ftp"
+JOB_METHOD_FTPS          = "ftps"
+JOB_METHOD_SFTP          = "sftp"
+JOB_METHOD_FILE          = "file"
+
+
 # Worker statuses
 WORKER_STATUS_STARTING  = "starting"
 WORKER_STATUS_WAITING   = "waiting"
@@ -66,7 +74,7 @@ WORKER_STATUS_TIMEOUT   = "timeout"
 WORKER_STATUS_CRASHED   = "crashed"
 WORKER_STATUS_CLEANING  = "cleaning"
 WORKER_STYLES = {
-  WORKER_STATUS_WAITING  => :success,
+  WORKER_STATUS_WAITING  => nil,
   WORKER_STATUS_RUNNING  => :info,
   WORKER_STATUS_CRASHED  => :danger,
   WORKER_STATUS_FINISHED => :success,

data/lib/rest-ftp-daemon/{stats.rb → counters.rb}

@@ -1,7 +1,7 @@
 module RestFtpDaemon
 
   # Queue that stores all the Jobs waiting to be processed or fully processed
-  class Stats
+  class Counters
     attr_reader :stats
 
     if Settings.newrelic_enabled?
@@ -22,7 +22,7 @@ module RestFtpDaemon
 
     def get group, name
       @mutex_stats.synchronize do
-        @stats[group][name] if @stats[group]
+        @stats[group][name] if @stats[group].is_a? Hash
       end
     end
 

data/lib/rest-ftp-daemon/helpers.rb

@@ -78,11 +78,11 @@ module RestFtpDaemon
     def self.job_method_label method
       return if method.nil?
       klass = case method
-      when :file
+      when JOB_METHOD_FILE
         "label-primary"
-      when :ftp
+      when JOB_METHOD_FTP
         "label-warning"
-      when :ftps, :ftpes
+      when JOB_METHOD_FTPS
         "label-success"
       else
         "label-default"

data/lib/rest-ftp-daemon/job.rb

@@ -19,7 +19,6 @@ module RestFtpDaemon
     attr_reader :error
     attr_reader :status
     attr_reader :runs
-    attr_reader :target_method
 
     attr_reader :queued_at
     attr_reader :updated_at
@@ -213,13 +212,6 @@ module RestFtpDaemon
       end
     end
 
-    def get attribute
-      @mutex.synchronize do
-        @infos || {}
-        @infos[attribute]
-      end
-    end
-
     def weight
       @weight = [
         - @runs.to_i,
@@ -247,12 +239,12 @@ module RestFtpDaemon
     end
 
     def targethost
-      get(:target_host)
+      get_info :target, :host
     end
 
-    def json_target
-      utf8 @target_method
-    end
+    # def json_target
+    #   utf8 get_info(:target, :method)
+    # end
 
     def json_error
       utf8 @error unless @error.nil?
@@ -262,6 +254,13 @@ module RestFtpDaemon
       utf8 @status unless @status.nil?
     end
 
+    def get_info level1, level2
+      @mutex.synchronize do
+        @infos || {}
+        @infos[level1][level2] if @infos[level1].is_a? Hash
+      end
+    end
+
   protected
 
     def expand_path path
@@ -304,21 +303,19 @@ module RestFtpDaemon
       @runs += 1
 
       # Init
-      @source_method = :file
-      @target_method = nil
       @source_path = nil
 
       # Prepare source
       raise RestFtpDaemon::JobMissingAttribute unless @source
       @source_path = expand_path @source
-      set_info :source_path, @source_path
-      set_info :source_method, :file
+      set_info :source, :path, @source_path
+      set_info :source, :method, JOB_METHOD_FILE
 
       # Prepare target
       raise RestFtpDaemon::JobMissingAttribute unless @target
       target_uri = expand_url @target
-      set_info :target_uri, target_uri.to_s
-      set_info :target_host, target_uri.host
+      set_info :target, :uri, target_uri.to_s
+      set_info :target, :host, target_uri.host
       @target_path = Path.new target_uri.path, true
 
       #puts "@target_path: #{@target_path.inspect}"
@@ -328,20 +325,23 @@ module RestFtpDaemon
 
       if target_uri.is_a? URI::FTP
         log_info "Job.prepare target_method FTP"
-        # set_info :target_method, :ftp
-        @target_method = :ftp
+        # set_info :target, :method, :ftp
+        set_info :target, :method, JOB_METHOD_FTP
+        #@target_method = :ftp
         @remote = RemoteFTP.new target_uri, log_context
 
       elsif (target_uri.is_a? URI::FTPES) || (target_uri.is_a? URI::FTPS)
         log_info "Job.prepare target_method FTPES"
-        # set_info :target_method, :ftpes
-        @target_method = :ftpes
+        # set_info :target, :method, :ftpes
+        set_info :target, :method, JOB_METHOD_FTPS
+        # @target_method = :ftpes
         @remote = RemoteFTP.new target_uri, log_context, ftpes: true
 
       elsif target_uri.is_a? URI::SFTP
         log_info "Job.prepare target_method SFTP"
-        # set_info :target_method, :sftp
-        @target_method = :sftp
+        # set_info :target, :method, :sftp
+        set_info :target, :method, JOB_METHOD_SFTP
+        # @target_method = :sftp
         @remote = RemoteSFTP.new target_uri, log_context
 
       else
@@ -360,13 +360,13 @@ module RestFtpDaemon
       raise RestFtpDaemon::JobAssertionFailed, "run/1" unless @source_path
       raise RestFtpDaemon::JobAssertionFailed, "run/2" unless @target_path
       @transfer_sent = 0
-      set_info :source_processed, 0
+      set_info :source, :processed, 0
 
       # Guess source files from disk
       set_status JOB_STATUS_CHECKING_SRC
       sources = find_local @source_path
-      set_info :source_count, sources.count
-      set_info :source_files, sources.collect(&:full)
+      set_info :source, :count, sources.count
+      set_info :source, :files, sources.collect(&:full)
       log_info "Job.run sources #{sources.collect(&:name)}"
       raise RestFtpDaemon::JobSourceNotFound if sources.empty?
 
@@ -383,7 +383,7 @@ module RestFtpDaemon
 
       # Compute total files size
       @transfer_total = sources.collect(&:size).sum
-      set_info :transfer_total, @transfer_total
+      set_info :transfer, :total, @transfer_total
 
       # Reset counters
       @last_data = 0
@@ -404,7 +404,7 @@ module RestFtpDaemon
         remote_push source, full_target
 
         # Update counters
-        set_info :source_processed, source_processed += 1
+        set_info :source, :processed, source_processed += 1
       end
 
       # FTP transfer finished
@@ -434,18 +434,18 @@ module RestFtpDaemon
       Thread.current.thread_variable_set :updated_at, now
     end
 
-    def set_info attribute, value
+    def set_info level1, level2, value
       @mutex.synchronize do
         @infos || {}
+        @infos[level1] ||= {}
 
         # Force strings to UTF8
         if value.is_a? Symbol
-          @infos[attribute] = value.to_s.force_encoding(Encoding::UTF_8)
+          @infos[level1][level2] = value.to_s.force_encoding(Encoding::UTF_8)
         elsif value.is_a? String
-          # @infos[attribute] = utf8(value)
-          @infos[attribute] = value.force_encoding(Encoding::UTF_8)
+          @infos[level1][level2] = value.dup.force_encoding(Encoding::UTF_8)
         else
-          @infos[attribute] = value
+          @infos[level1][level2] = value
         end
 
         # Mark the job as updated
@@ -490,9 +490,9 @@ module RestFtpDaemon
       set_status JOB_STATUS_DISCONNECTING
       @finished_at = Time.now
 
-      # Update stats
-      $stats.increment :jobs, :finished
-      $stats.add :global, :transferred, @transfer_total
+      # Update counters
+      $counters.increment :jobs, :finished
+      $counters.add :data, :transferred, @transfer_total
     end
 
     def remote_push source, target
@@ -503,7 +503,7 @@ module RestFtpDaemon
 
       # Use source filename if target path provided none (typically with multiple sources)
       log_info "Job.remote_push [#{source.name}]: [#{source.full}] > [#{target.full}]"
-      set_info :source_current, source.name
+      set_info :source, :current, source.name
 
       # Compute temp target name
       tempname = nil
@@ -537,10 +537,10 @@ module RestFtpDaemon
 
       # Compute final bitrate
       global_transfer_bitrate = get_bitrate @transfer_total, (Time.now - transfer_started_at)
-      set_info :transfer_bitrate, global_transfer_bitrate.round(0)
+      set_info :transfer, :bitrate, global_transfer_bitrate.round(0)
 
       # Done
-      set_info :source_current, nil
+      set_info :source, :current, nil
     end
 
     def progress transferred, name = ""
@@ -549,17 +549,17 @@ module RestFtpDaemon
 
       # Update counters
       @transfer_sent += transferred
-      set_info :transfer_sent, @transfer_sent
+      set_info :transfer, :sent, @transfer_sent
 
       # Update job info
       percent0 = (100.0 * @transfer_sent / @transfer_total).round(0)
-      set_info :progress, percent0
+      set_info :transfer, :progress, percent0
 
       # Update job status after each NOTIFY_UPADE_STATUS
       progressed_ago = (now.to_f - @progress_at.to_f)
       if (!JOB_UPDATE_INTERVAL.to_f.zero?) && (progressed_ago > JOB_UPDATE_INTERVAL.to_f)
         @current_bitrate = running_bitrate @transfer_sent
-        set_info :transfer_bitrate, @current_bitrate.round(0)
+        set_info :transfer, :bitrate, @current_bitrate.round(0)
 
         # Log progress
         stack = []
@@ -644,21 +644,21 @@ module RestFtpDaemon
       # Update job's internal status
       set_status JOB_STATUS_FAILED
       set_error error
-      set_info :error_exception, exception.class.to_s
-      set_info :error_message, exception.message
+      set_info :error, :exception, exception.class.to_s
+      set_info :error, :message, exception.message
 
       # Build status stack
       notif_status = nil
       if include_backtrace
-        set_info :error_backtrace, exception.backtrace
+        set_info :error, :backtrace, exception.backtrace
         notif_status = {
           backtrace: exception.backtrace,
           }
       end
 
       # Increment counter for this error
-      $stats.increment :errors, error
-      $stats.increment :jobs, :failed
+      $counters.increment :errors, error
+      $counters.increment :jobs, :failed
 
       # Prepare notification if signal given
       return unless event

data/lib/rest-ftp-daemon/job_queue.rb

@@ -16,16 +16,13 @@ module RestFtpDaemon
       # Instance variables
       @queues = {}
       @waitings = {}
-
-      # @queue = []
-      # @waiting = []
-
       @jobs = []
 
       @queues.taint          # enable tainted communication
       @waitings.taint
-
       taint
+
+      # Global mutex for the queue
       @mutex = Mutex.new
 
       # Logger
@@ -44,30 +41,42 @@ module RestFtpDaemon
       prefixed_id @last_id
     end
 
-      def jobs_queued
+    def jobs_queued
       @queues
-      #@queues.map { |status, jobs| jobs.size }
     end
 
-    # Statistics on average rates
     def rate_by method_name
       # Init
-      rates = {}
+      result = {}
       return unless Job.new(0, {}).respond_to? method_name
 
-      # Group jobs by method_name
-      jobs_grouped = @jobs.group_by do |job|
-        job.send(method_name)
-      end
+      # Select only running jobs
+      @jobs.each do |job|
+
+        # Compute jobs's group, next if empty
+        group = job.send(method_name)
+        next if group.nil?
 
-      # Inside each group, sum up rates for interesting statuses
-      jobs_grouped.map do |group, jobs|
-        # Store their sum
-        rates[group] = rates_by_status (jobs)
+        # Initialize rate entry
+        result[group] ||= nil
+
+        # If job is not uploading, next !
+        next unless job.status == JOB_STATUS_UPLOADING
+
+        # Extract current rate, next if not available
+        rate = job.get_info :transfer, :bitrate
+        next if rate.nil?
+
+
+        # Add its current rate
+        result[group] ||= 0
+        #log_info "  2: #{result.inspect} (rate: #{rate})"
+        result[group] += rate
+        #log_info "  3: #{result.inspect}"
       end
 
       # Return the rate
-      rates
+      result
     end
 
     # Queue infos
@@ -80,20 +89,18 @@ module RestFtpDaemon
       @jobs.group_by { |job| job.status }.map { |status, jobs| statuses[status] = jobs.size }
       statuses
     end
-    alias jobs_count_by_status jobs_by_status
 
-    def jobs_ids
-      @jobs.collect(&:id)
-    end
+    # def jobs_ids
+    #   @jobs.collect(&:id)
+    # end
 
     def empty?
       @queue.empty?
     end
 
-    def num_waiting
-      @waiting.size
-    end
-
+    # def num_waiting
+    #   @waiting.size
+    # end
 
     # Queue access
     def find_by_id id, prefixed = false
@@ -176,7 +183,7 @@ module RestFtpDaemon
 
       # Status filtering: only those jobs
       else
-        @jobs.select { |job| job.status.to_s == status.to_s }
+        @jobs.select { |job| job.status == status.to_s }
 
       end
     end
@@ -222,26 +229,6 @@ module RestFtpDaemon
       "#{@prefix}.#{id}"
     end
 
-    def rates_by_status jobs
-      rates = {}
-
-      # Sub-group by status
-      jobs.group_by(&:status).each do |status, jobset|
-
-        # Extract bitrate values
-        bitrates = jobset.collect do |job|
-          job.get(:transfer_bitrate)
-        end
-
-        # Store their sum
-        rates[status] = bitrates.reject(&:nil?).sum
-        # rates["#{status}_ids"] = jobset.collect(&:id).join(', ')
-      end
-
-      # Return rates
-      rates
-    end
-
     if Settings.newrelic_enabled?
       add_transaction_tracer :push,                 category: :task
       add_transaction_tracer :pop,                  category: :task

data/lib/rest-ftp-daemon/logger.rb

@@ -15,13 +15,10 @@ class Logger
 
     if lines.is_a? Hash
       output = build_from_hash prefix2, lines
-
     elsif lines.is_a? Array
       output = build_from_array prefix2, lines
-
     else
       output = []
-
     end
 
     # Prepend plain message to output

data/lib/rest-ftp-daemon/paginate.rb

@@ -52,9 +52,7 @@ module RestFtpDaemon
 
     def link p
       klass = (p == @page)? "primary" : "default"
-
       url = Helpers.dashboard_filter_url(@filter)
-
       "<a class='btn btn-%s' href='%s?page=%d'>%p</a>" % [
         klass,
         @filter,

data/lib/rest-ftp-daemon/remote.rb

@@ -31,10 +31,6 @@ module RestFtpDaemon
 
   private
 
-    # def log_context
-    #   @log_context
-    # end
-
     def myname
       self.class.to_s
     end

data/lib/rest-ftp-daemon/remote_ftp.rb

@@ -43,7 +43,6 @@ module RestFtpDaemon
       log_info "RemoteFTP.present? [#{target.name}]"
 
     rescue Net::FTPPermError
-      # log_info "RemoteFTP.present? [#{target.name}] NOT_FOUND"
       return false
     else
       return size
@@ -102,9 +101,6 @@ module RestFtpDaemon
       log_info "RemoteFTP.push to [#{destination.name}]"
 
       @ftp.putbinaryfile source.full, target.name, @chunk_size do |data|
-        # Update the worker activity marker
-        # FIXME: worker_is_still_active
-
         # Update job status after this block transfer
         yield data.bytesize, destination.name
       end
@@ -134,6 +130,5 @@ module RestFtpDaemon
       @ftp.ftps_mode = DoubleBagFTPS::EXPLICIT
     end
 
-
   end
 end

data/lib/rest-ftp-daemon/remote_sftp.rb

@@ -123,9 +123,6 @@ module RestFtpDaemon
           # args[2] : data being written (as string)
           # puts "writing #{args[2].length} bytes to #{args[0].remote} starting at #{args[1]}"
 
-          # Update the worker activity marker
-          # FIXME: worker_is_still_active
-
           # Update job status after this block transfer
           yield args[2].length, destination.name
 
@@ -154,8 +151,6 @@ module RestFtpDaemon
     def close
       # Close init
       super
-
-      # @sftp.close
     end
 
     def connected?

data/lib/rest-ftp-daemon/settings.rb

@@ -48,7 +48,6 @@ class Settings < Settingslogic
       ENV["NEWRELIC_AGENT_ENABLED"] = "false"
     end
 
-    # That's it!
   end
 
   def overwrite options

data/lib/rest-ftp-daemon/views/dashboard.haml

@@ -14,11 +14,7 @@
       .container-fluid
         = render :dashboard_header
 
-
     .container-fluid
-
-      %br
-
       .row
         #box-jobs.col-md-12
           = render :dashboard_jobs
@@ -29,13 +25,11 @@
 
         #box-workers.col-md-3
           = render :dashboard_workers
-          = render :dashboard_stats
 
         #box-rates.col-md-3
+          = render :dashboard_counters
           = render :dashboard_rates
 
-
     .footer
       .container-fluid
         = render :dashboard_footer
-

data/lib/rest-ftp-daemon/views/{dashboard_stats.haml → dashboard_counters.haml}

@@ -1,5 +1,5 @@
 -# coding: utf-8
-%h2 Stats
+%h2 Counters
 
 %table.table.table-striped.table-hover.table-condensed
 
@@ -11,11 +11,10 @@
 
   %tbody
 
-    - $stats.stats.each do |group, values|
+    - $counters.stats.each do |group, values|
 
       - values.each do |name, value|
         %tr
           %td= group
           %td= name
           %td.text-right= value
-

data/lib/rest-ftp-daemon/views/dashboard_footer.haml

@@ -1,7 +1,6 @@
 -# coding: utf-8
 - info_procs = (Facter.value :processorcount).to_i
 
-
 .footer-signature.pull-left
 
   ©
@@ -31,5 +30,3 @@
   .btn-group.btn-group-sm
     .btn.btn-default.btn-info Started
     .btn.btn-default= Helpers.datetime_short(APP_STARTED)
-
-

data/lib/rest-ftp-daemon/views/dashboard_header.haml

@@ -2,8 +2,8 @@
 - info_procs = (Facter.value :processorcount).to_i
 - info_load = Sys::CPU.load_avg.first.to_f
 - info_norm = info_procs.zero? ? "N/A" : (100 * info_load / info_procs).round(1)
-- info_trans = $stats.get :global, :transferred
-- info_processed = $stats.get(:global, :processed)
+- info_trans = $counters.get :data, :transferred
+- info_processed = $counters.get :jobs, :processed
 - mem = GetProcessMem.new
 
 
@@ -36,5 +36,3 @@
   .btn-group.btn-group-sm
     .btn.btn-default.btn-success Transferred
     .btn.btn-default= Helpers.format_bytes(info_trans, "B", 1)
-
-

data/lib/rest-ftp-daemon/views/dashboard_jobs.haml

@@ -1,5 +1,5 @@
 -# coding: utf-8
-- jobs_count_by_status = $queue.jobs_count_by_status
+- jobs_by_status = $queue.jobs_by_status
 - counts_all = $queue.jobs_count
 - jobs = @paginate.subset
 
@@ -9,7 +9,7 @@
     ALL (#{counts_all})
 
 .btn-group.btn-group-xs.filters
-  - jobs_count_by_status.each do |status, count|
+  - jobs_by_status.each do |status, count|
     - klass = (status.to_s == @filter) ? "btn-info" : ""
     %a.btn.btn-default{href: Helpers.dashboard_filter_url(status), class: klass}
       #{status} (#{count})
@@ -33,7 +33,7 @@
           %th label
           %th pool
           %th source
-          %th mode
+          %th method
           %th target
           %th queued
           %th.error status
@@ -44,7 +44,6 @@
           %th{title: "Priority"} P
           %th{title: "Runs count"} R
 
-
         - @jobs_queued.each do |pool, jobs|
           - unless jobs.empty?
             %tbody.jobs
@@ -59,5 +58,3 @@
           %tbody.jobs
             = render :dashboard_table, {jobs: jobs}
 
-
-

data/lib/rest-ftp-daemon/views/dashboard_rates.haml

@@ -13,24 +13,20 @@
     %thead
       %tr
         %th= group_title
-        %th.text-right=JOB_STATUS_FINISHED
-        %th.text-right=JOB_STATUS_UPLOADING
+        %th.text-right bitrate
 
     %tbody
-      - rates_by_status.each do |group, rates|
+      - rates_by_status.each do |group, rate|
         - next if group.nil?
+        - trclass = (rate.nil? ? "" : "info")
 
-        %tr.info
-          %td= group
+        %tr{class: trclass}
+          %td
+            = group
           %td.text-right
-            = Helpers.format_bytes(rates[JOB_STATUS_FINISHED], "bps")
-          %td.text-right
-            = Helpers.format_bytes(rates[JOB_STATUS_UPLOADING], "bps")
-          -# %td= rates.inspect
+            = Helpers.format_bytes(rate, "bps")
 
     %thead
       %tr
         %td{colspan: 3}
           %br
-
-    -# JOB_STATUS_UPLOADING

data/lib/rest-ftp-daemon/views/dashboard_table.haml

@@ -1,10 +1,10 @@
 -# coding: utf-8
 - jobs.each do |job|
-  - progress = job.get :progress
-  - source_count = job.get(:source_count) || 0
-  - source_processed = job.get(:source_processed) || 0
-  - source_current = job.get(:source_current)
-  - bitrate = job.get :transfer_bitrate
+  - progress = job.get_info(:transfer, :progress)
+  - source_count = job.get_info(:source, :count) || 0
+  - source_processed = job.get_info(:source, :processed) || 0
+  - source_current = job.get_info(:source, :current)
+  - bitrate = job.get_info(:transfer, :bitrate)
   - trclass = JOB_STYLES[job.status]
   - runs = job.runs.to_i
 
@@ -20,14 +20,14 @@
 
     %td= job.pool
 
-    %td{title: job.get(:source_path)}
+    %td{title: job.get_info(:source, :path)}
       = Helpers.highlight_tokens job.source
 
     %td
-      =# Helpers.job_method_label (job.get :target_method)
-      = Helpers.job_method_label job.target_method
+      = Helpers.job_method_label (job.get_info :target, :method)
+      =# Helpers.job_method_label job.target_method
 
-    %td{title: job.get(:target_url)}
+    %td{title: job.get_info(:target, :url)}
       = Helpers.highlight_tokens job.target
 
     %td
@@ -54,14 +54,14 @@
       - if job.status == JOB_STATUS_UPLOADING
         .progress
           .progress-bar{style:"width: #{progress}%;"}
-            = Helpers.format_bytes(job.get(:transfer_sent), "B")
+            = Helpers.format_bytes(job.get_info(:transfer, :sent), "B")
 
       - else
-        .error{title: job.get(:error_message)}
+        .error{title: job.get_info(:error, :message)}
           = Helpers.text_or_empty(job.error)
 
     %td.nobr.text-right
-      = Helpers.format_bytes(job.get(:transfer_total), "B")
+      = Helpers.format_bytes(job.get_info(:transfer, :total), "B")
 
     %td.nobr.text-right{title: "time: #{job.exectime} s"}
       - if bitrate
@@ -78,5 +78,3 @@
 
     %td
       .label.flag.worker-label{class: Helpers.job_runs_style(runs)}= runs
-
-

data/lib/rest-ftp-daemon/views/dashboard_workers.haml

@@ -1,4 +1,6 @@
 -# coding: utf-8
+-  variables = $pool.worker_variables
+
 %h2 Workers
 
 %table.table.table-striped.table-hover.table-condensed
@@ -12,7 +14,7 @@
       %th.text-right seen
 
   %tbody
-    - @worker_variables.each do |vars|
+    - variables.each do |vars|
       - wid = vars[:wid]
       - status = vars[:status]
       - alive = $pool.worker_alive? wid
@@ -22,7 +24,6 @@
         - trclass = "danger"
         - status = "DEAD"
 
-
       %tr{class: trclass.to_s}
         %td= vars[:pool]
         %td= wid

data/lib/rest-ftp-daemon/worker.rb

@@ -13,7 +13,6 @@ module RestFtpDaemon
       @log_worker_status_changes = true
 
       # Worker name
-      #@wid = wid
       @pool = pool
 
       # Set thread context

data/lib/rest-ftp-daemon/worker_job.rb

@@ -35,8 +35,6 @@ module RestFtpDaemon
       max_age = Settings.at(:retry, :max_age)
       max_runs = Settings.at(:retry, :max_runs)
       delay = Settings.at(:retry, :delay)
-      # log_error "on_errors: #{on_errors.inspect} (#{on_errors.first.class})"
-      # log_error "job.error: #{job.error} (#{job.error.class})"
 
       if !job.error
         #log_info "job succeeded"
@@ -86,7 +84,7 @@ module RestFtpDaemon
       worker_status WORKER_STATUS_FINISHED, job
 
       # Increment total processed jobs count
-      $stats.increment :global, :processed
+      $counters.increment :jobs, :processed
 
     rescue RestFtpDaemon::JobTimeout => ex
       log_error "JOB TIMED OUT", ex.backtrace

data/rest-ftp-daemon.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rubocop", "~> 0.32.0"
   spec.add_development_dependency "pry"
 
-  spec.add_runtime_dependency "thin", "~> 1.6.4"
+  spec.add_runtime_dependency "thin", "~> 1.6"
   spec.add_runtime_dependency "grape"
   spec.add_runtime_dependency "grape-entity"
   spec.add_runtime_dependency "settingslogic"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: 0.250.0
+  version: 0.250.3
 platform: ruby
 authors:
 - Bruno MEDICI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-19 00:00:00.000000000 Z
+date: 2016-05-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -100,14 +100,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.4
+        version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.4
+        version: '1.6'
 - !ruby/object:Gem::Dependency
   name: grape
   requirement: !ruby/object:Gem::Requirement
@@ -291,6 +291,7 @@ files:
 - lib/rest-ftp-daemon/api/status.rb
 - lib/rest-ftp-daemon/array.rb
 - lib/rest-ftp-daemon/constants.rb
+- lib/rest-ftp-daemon/counters.rb
 - lib/rest-ftp-daemon/exceptions.rb
 - lib/rest-ftp-daemon/helpers.rb
 - lib/rest-ftp-daemon/job.rb
@@ -308,14 +309,13 @@ files:
 - lib/rest-ftp-daemon/settings.rb
 - lib/rest-ftp-daemon/static/css/bootstrap.css
 - lib/rest-ftp-daemon/static/css/main.css
-- lib/rest-ftp-daemon/stats.rb
 - lib/rest-ftp-daemon/uri.rb
 - lib/rest-ftp-daemon/views/dashboard.haml
+- lib/rest-ftp-daemon/views/dashboard_counters.haml
 - lib/rest-ftp-daemon/views/dashboard_footer.haml
 - lib/rest-ftp-daemon/views/dashboard_header.haml
 - lib/rest-ftp-daemon/views/dashboard_jobs.haml
 - lib/rest-ftp-daemon/views/dashboard_rates.haml
-- lib/rest-ftp-daemon/views/dashboard_stats.haml
 - lib/rest-ftp-daemon/views/dashboard_table.haml
 - lib/rest-ftp-daemon/views/dashboard_tokens.haml
 - lib/rest-ftp-daemon/views/dashboard_workers.haml