rest-ftp-daemon 0.433.0 → 0.434.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9e668f8f43effe91b3332111a2963abb1011343c
-  data.tar.gz: 79bc3e98795a91cdef80609096b15fc9606f4a6a
+  metadata.gz: 591f494a7bf660ee944096abcb623bc0d771157a
+  data.tar.gz: 34f150dc4b22573c0cbc948788adee9618f82599
 SHA512:
-  metadata.gz: 0a06749631bb61dd15e9ef0556e8909fd6ca75ab32d944e70a854533a8caa9e50854fd13942b2fa11dda2dfea2f098dbf15c9fdd2e123c00971911f35f7ce396
-  data.tar.gz: c6ae3a6914a80a1c932947f63be90d8d9a722134411839cdfa6d0694e8c3555ac86dac8b9afdbc923561489c0993186c61811d49b375a981f8fa4dbae0b8513a
+  metadata.gz: 087935f7b3179f7033f233e3344d98e26e6ad00e8e192722991409cc2d86f320857d7a87272d70812c6a34144e4f9b668b36b2a16f624fb2c00011478d7f914f
+  data.tar.gz: 58807f103079e56eb17b679b6d5d29552aea8d30d417c7b9b2dc0a46237b23bf1b4c9d3191a5a906a872ab36798ad698f6d1d2259f5031d15363eecc3ac3cd90

data/Gemfile.lock

@@ -1,11 +1,11 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.433.0)
+    rest-ftp-daemon (0.434.0)
       activesupport (~> 4.2)
       api-auth
       aws-sdk-resources (~> 2)
-      bmc-daemon-lib (~> 0.3.12)
+      bmc-daemon-lib (~> 0.3.18)
       double-bag-ftps
       facter
       get_process_mem
@@ -37,17 +37,15 @@ GEM
     addressable (2.4.0)
     api-auth (2.0.1)
     ast (2.3.0)
-    astrolabe (1.3.1)
-      parser (~> 2.2)
-    aws-sdk-core (2.5.11)
+    aws-sdk-core (2.6.1)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.5.11)
-      aws-sdk-core (= 2.5.11)
+    aws-sdk-resources (2.6.1)
+      aws-sdk-core (= 2.6.1)
     axiom-types (0.1.1)
       descendants_tracker (~> 0.0.4)
       ice_nine (~> 0.11.0)
       thread_safe (~> 0.3, >= 0.3.1)
-    bmc-daemon-lib (0.3.12)
+    bmc-daemon-lib (0.3.18)
       chamber (~> 2.9)
     builder (3.2.2)
     chamber (2.9.1)
@@ -95,7 +93,7 @@ GEM
       representable
     haml (4.0.7)
       tilt
-    hashie (3.4.4)
+    hashie (3.4.6)
     http (2.0.3)
       addressable (~> 2.3)
       http-cookie (~> 1.0)
@@ -121,7 +119,7 @@ GEM
     net-ssh (3.2.0)
     netrc (0.11.0)
     newrelic_rpm (3.16.2.321)
-    parser (2.3.1.2)
+    parser (2.3.1.3)
       ast (~> 2.2)
     powerpack (0.1.1)
     pry (0.10.4)
@@ -140,7 +138,7 @@ GEM
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 3.0)
       netrc (~> 0.7)
-    rollbar (2.12.0)
+    rollbar (2.13.0)
       multi_json
     rspec (3.5.0)
       rspec-core (~> 3.5.0)
@@ -155,12 +153,13 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.5.0)
     rspec-support (3.5.0)
-    rubocop (0.32.1)
-      astrolabe (~> 1.3)
-      parser (>= 2.2.2.5, < 3.0)
+    rubocop (0.43.0)
+      parser (>= 2.3.1.1, < 3.0)
       powerpack (~> 0.1)
       rainbow (>= 1.99.1, < 3.0)
-      ruby-progressbar (~> 1.4)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 1.0, >= 1.0.1)
+    ruby-prof (0.16.2)
     ruby-progressbar (1.8.1)
     slop (3.6.0)
     streamio-ffmpeg (3.0.0)
@@ -180,6 +179,7 @@ GEM
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.2)
+    unicode-display_width (1.1.1)
     virtus (1.0.5)
       axiom-types (~> 0.1)
       coercible (~> 1.0)
@@ -196,7 +196,8 @@ DEPENDENCIES
   rake
   rest-ftp-daemon!
   rspec
-  rubocop (~> 0.32.0)
+  rubocop
+  ruby-prof
 
 BUNDLED WITH
    1.12.5

data/bin/rest-ftp-daemon

@@ -91,7 +91,7 @@ end
 
 
 # Thin: basic options
-thin_logfile = Conf.logfile_path(:thin)
+thin_logfile = Conf.logfile(:thin)
 
 argv = []
 argv << ["-e",          Conf.app_env]

data/config.ru

@@ -1,12 +1,12 @@
 # Load gem files
-Conf.log :init, "rackup: project code"
+Conf.log :rackup, "load project code"
 
 load_path_libs = File.expand_path "lib", File.dirname(__FILE__)
 $LOAD_PATH.unshift(load_path_libs) unless $LOAD_PATH.include?(load_path_libs)
 require "rest-ftp-daemon"
 
 # Rack authent
-Conf.log :init, "rackup: rack setup"
+Conf.log :rackup, "authentication, assets, env-specific"
 unless Conf[:adminpwd].nil?
   use Rack::Auth::Basic, "Restricted Area" do |username, password|
     [username, password] == ["admin", Conf[:adminpwd]]
@@ -23,12 +23,20 @@ use Rack::Static, root: "#{Conf.app_libs}/static/", urls: [
   MOUNT_SWAGGER_UI,
   ]
 
-# Rack reloader
-if Conf.app_env == "development"
-  # use Rack::Reloader, 1
+# Env-specific configuration
+case Conf.app_env
+  when "development"
+    # Rack reloader
+    # use Rack::Reloader, 1
+
+    # Newrelic dev mode
+    require 'new_relic/rack/developer_mode'
+    use NewRelic::Rack::DeveloperMode
+
+  when "production"
 end
 
 # Launch the API
-Conf.log :init, "rackup: start endpoint"
+Conf.log :rackup, "start API::Root"
 run RestFtpDaemon::API::Root
 

data/lib/rest-ftp-daemon/api/root.rb

@@ -48,7 +48,7 @@ module RestFtpDaemon
 
       ## EXCEPTION HANDLERS
       rescue_from :all do |exception|
-        Rollbar.error "#{exception.class.name}: #{exception.message}"
+        Rollbar.error exception, "api [#{error}]: #{exception.class.name}: #{exception.message}"
         #error!({error: :internal_server_error, message: exception.message}, 500)
         exception_error :api_unexpected_error, 500, exception
       end

data/lib/rest-ftp-daemon/entities/job.rb

@@ -30,16 +30,19 @@ module RestFtpDaemon
       # Status and error
       expose :status, format_with: :utf8_filter
       expose :error, format_with: :utf8_filter
+      expose :tentatives
 
       # Time stamps
-      expose :queued_at
-      expose :updated_at
+      expose :created_at
+      expose :created_since       #, safe: true
+
       expose :started_at
+      expose :started_since
+
       expose :finished_at
+      expose :finished_in
 
-      # Computed fields
-      expose :age           #, safe: true
-      expose :exectime
+      expose :updated_at
 
       # Infos
       expose :infos, unless: :hide_infos

data/lib/rest-ftp-daemon/initialize.rb

@@ -1,17 +1,15 @@
 # Init Rollbar and Newrelic
-Conf.log :init, "init: Newrelic and Rollbar"
 Conf.prepare_newrelic
 Conf.prepare_rollbar
 
 # Initialize Facter
-Conf.log :init, "init: Facter"
+Conf.log :initialize, "prepare Facter"
 begin
   Facter.loadfacts
 rescue StandardError => exception
-  Conf.log :init, "init: Facter failed to initialize: #{exception.message}"
+  Conf.log :initialize, "facter failed to initialize: #{exception.message}"
 end
 
 # Initialize workers
-Conf.log :init, "init: workers"
+Conf.log :initialize, "prepare workers"
 RestFtpDaemon::WorkerPool.instance.start_em_all
-

data/lib/rest-ftp-daemon/job.rb

@@ -28,11 +28,15 @@ module RestFtpDaemon
     attr_reader :status
     attr_reader :tentatives
 
-    attr_reader :queued_at
+    attr_reader :created_at
     attr_reader :updated_at
     attr_reader :started_at
     attr_reader :finished_at
 
+    attr_reader :created_since
+    attr_reader :started_since
+    attr_reader :finished_in
+
     attr_reader :infos
 
     # Define readers from imported fields
@@ -50,14 +54,15 @@ module RestFtpDaemon
 
       # Init context
       @id           = job_id.to_s
-      @started_at   = nil
-      @finished_at  = nil
       @updated_at   = nil
       @error        = nil
       @status       = nil
-      @tentatives         = 0
+      @tentatives   = 0
       @wid          = nil
 
+      # Update timestamps
+      @created_at   = Time.now
+
       # Prepare configuration
       @config       = Conf[:transfer] || {}
       @endpoints    = Conf[:endpoints] || {}
@@ -91,10 +96,9 @@ module RestFtpDaemon
 
       # Increment run cours
       @tentatives +=1
-
-      # Flag current job timestamps
-      @queued_at = Time.now
       @updated_at = Time.now
+      @started_at   = nil
+      @finished_at  = nil
 
       # Job has been prepared, reset infos
       set_status JOB_STATUS_PREPARED
@@ -108,40 +112,43 @@ module RestFtpDaemon
     end
 
     # Process job
-    def process
+    def start
       # Check prerequisites
       raise RestFtpDaemon::AssertionFailed, "run/source_loc" unless @source_loc
       raise RestFtpDaemon::AssertionFailed, "run/target_loc" unless @target_loc
 
+      # Remember when we started
+      @started_at = Time.now
+
       # Notify we start working
-      log_info "Job.process notify [started]"
+      log_info "Job.start notify [started]"
       current_signal = :started
       set_status JOB_STATUS_WORKING
       client_notify :started
 
       # Before work
-      log_debug "Job.process do_before"
+      log_debug "Job.start do_before"
       current_signal = :started
       do_before
 
       # Do the hard work
-      log_debug "Job.process do_work"
+      log_debug "Job.start do_work"
       current_signal = :ended
       do_work
 
       # Finalize all this
-      log_debug "Job.process do_after"
+      log_debug "Job.start do_after"
       current_signal = :ended
       do_after
 
     rescue StandardError => exception
-      Rollbar.error "Job.process: #{exception.class.name}: #{exception.message}"
+      Rollbar.error exception, "job [#{error}]: #{exception.class.name}: #{exception.message}"
       return oops current_signal, exception
 
     else
       # All done !
       set_status JOB_STATUS_FINISHED
-      log_info "JobVideo.process notify [ended]"
+      log_info "Job.start notify [ended]"
       client_notify :ended
     end
 
@@ -164,28 +171,26 @@ module RestFtpDaemon
       @weight = [
         - @tentatives.to_i,
         + @priority.to_i,
-        - @queued_at.to_i,
+        - @created_at.to_i,
         ]
     end
 
-    def exectime
-      return nil if @started_at.nil? || @finished_at.nil?
-      (@finished_at - @started_at).round(2)
+    def started_since
+      since @started_at
     end
 
-    def oops_after_crash exception
-      Rollbar.error "oops_after_crash: #{exception.class.name}: #{exception.message}"
-      oops :ended, exception, "crashed"
+    def created_since
+      since @created_at
     end
 
-    def oops_you_stop_now exception
-      Rollbar.error "oops_you_stop_now: #{exception.class.name}: #{exception.message}"
-      oops :ended, exception, "timeout"
+    def finished_in
+      return nil if @started_at.nil? || @finished_at.nil?
+      (@finished_at - @started_at).round(2)
     end
 
-    def age
-      return nil if @queued_at.nil?
-      (Time.now - @queued_at).round(2)
+    def oops_end what, exception
+      Rollbar.error exception, "oops_end [#{what}]: #{exception.class.name}: #{exception.message}"
+      oops :ended, exception, what
     end
 
     def targethost
@@ -225,6 +230,12 @@ module RestFtpDaemon
       Thread.current.thread_variable_set :updated_at, now
     end
 
+    # Timestamps calculation
+    def since timestamp
+      return nil if timestamp.nil?
+      return (Time.now - timestamp).round(2)
+    end
+
     # Force strings to UTF8
     def debug_value_utf8 value
       case value
@@ -298,7 +309,7 @@ module RestFtpDaemon
       end
 
       # Log to Rollbar
-      Rollbar.error(exception)
+      Rollbar.warning e, "oops [#{error}]: #{exception.class.name}: #{exception.message}"
 
       # Close ftp connexion if open
       @remote.close unless @remote.nil? || !@remote.connected?

data/lib/rest-ftp-daemon/views/dashboard_jobs.haml

@@ -32,7 +32,7 @@
           %th{title: "Job identifier"} ID
           %th{title: "Job type"} T
           %th pool
-          %th queued
+          %th created
           %th label
           %th source
           %th target

data/lib/rest-ftp-daemon/views/dashboard_table.haml

@@ -27,7 +27,9 @@
 
     %td= job.pool
 
-    %td= datetime_short(job.queued_at)
+    %td
+      %nobr
+        = datetime_short(job.created_at)
 
     %td= job.label
 
@@ -69,7 +71,7 @@
     %td.nobr.text-right
       = format_bytes(transfer_total, "B")
 
-    %td.nobr.text-right{title: "time: #{job.exectime} s"}
+    %td.nobr.text-right{title: "started since: #{job.started_since} s"}
       - if transfer_bitrate
         %span.push-bitrate
           = format_bytes(transfer_bitrate, "bps")

data/lib/rest-ftp-daemon/worker_pool.rb

@@ -94,7 +94,10 @@ module RestFtpDaemon
       log_info "spawning #{klass.name} wid[#{wid}] pool[#{pool}]"
       @workers[wid] = Thread.new do
         begin
+          # Create a worker inside
           worker = klass.new wid, pool
+
+          # FIXME: sleep for a small amount of time to allow correct sequence of logging
           sleep 0.1
         rescue StandardError => ex
           log_error "#{klass.name} EXCEPTION: #{ex.message}"

data/lib/rest-ftp-daemon/workers/reporter.rb

@@ -60,7 +60,8 @@ module RestFtpDaemon
       end
 
       # Don't dump metrics unless we're debugging
-      msg_newrelic = "reported metrics to NewRelic [#{ENV['NEW_RELIC_APP_NAME']}]"
+      newrelic_app_name = Conf.at(:newrelic, :app_name)
+      msg_newrelic = "reported metrics to NewRelic [#{newrelic_app_name}]"
       if @config[:debug]
         log_debug msg_newrelic, metrics_newrelic
       else

data/lib/rest-ftp-daemon/workers/transfer.rb

@@ -53,25 +53,25 @@ module RestFtpDaemon
 
       # Processs this job protected by a timeout
       Timeout.timeout(@config[:timeout], RestFtpDaemon::JobTimeout) do
-        job.process
+        job.start
       end
 
       # Increment total processed jobs count
       RestFtpDaemon::Counters.instance.increment :jobs, :processed
 
     rescue RestFtpDaemon::JobTimeout => ex
-      log_error "JOB TIMEOUT", ex.backtrace
+      log_error "JOB TIMEOUT started_at[#{job.started_at}] started_since[#{job.started_since}] #{ex.message}", ex.backtrace
       worker_status WORKER_STATUS_TIMEOUT, job
 
       # Inform the job
-      job.oops_you_stop_now ex unless job.nil?
+      job.oops_end(:timeout, ex) unless job.nil?
 
     rescue RestFtpDaemon::AssertionFailed, RestFtpDaemon::JobAttributeMissing, StandardError => ex
-      log_error "JOB EXCEPTION ex[#{ex.class}] #{ex.message}", ex.backtrace
+      log_error "JOB CRASHED ex[#{ex.class}] #{ex.message}", ex.backtrace
       worker_status WORKER_STATUS_CRASHED
 
       # Inform the job
-      job.oops_after_crash ex unless job.nil?
+      job.oops_end(:crashed, ex) unless job.nil?
     end
 
     def handle_job_result job
@@ -118,7 +118,7 @@ module RestFtpDaemon
       return false unless @config[:retry_for]
 
       # Job age above this limit
-      return job.age >= @config[:retry_for]
+      return job.created_since >= @config[:retry_for]
     end
 
     def error_reached_max job

data/rest-ftp-daemon.gemspec

@@ -2,7 +2,7 @@
 Gem::Specification.new do |spec|
 
   # Project version
-  spec.version                    = "0.433.0"
+  spec.version                    = "0.434.0"
 
   # Project description
   spec.name                       = "rest-ftp-daemon"
@@ -26,12 +26,13 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
-  spec.add_development_dependency "rubocop", "~> 0.32.0"
+  spec.add_development_dependency "rubocop"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "http"
+  spec.add_development_dependency "ruby-prof"
 
   # Runtime dependencies
-  spec.add_runtime_dependency     "bmc-daemon-lib", "~> 0.3.12"
+  spec.add_runtime_dependency     "bmc-daemon-lib", "~> 0.3.18"
   spec.add_runtime_dependency     "json", "~> 1.8"
   spec.add_runtime_dependency     "thin", "~> 1.7"
   spec.add_runtime_dependency     "activesupport", "~> 4.2"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: 0.433.0
+  version: 0.434.0
 platform: ruby
 authors:
 - Bruno MEDICI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-12 00:00:00.000000000 Z
+date: 2016-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -56,16 +56,16 @@ dependencies:
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.32.0
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.32.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -94,20 +94,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-prof
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bmc-daemon-lib
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.12
+        version: 0.3.18
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.12
+        version: 0.3.18
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement