rest-ftp-daemon 0.424.0 → 0.424.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2d4d6f823c82eb53007aceccb525111fce662019
-  data.tar.gz: b779d0afa8de9f3f4cf3e935574223f24883e331
+  metadata.gz: b7e975d417576100041ffbdaf11a22f811a1e585
+  data.tar.gz: 27cb0f4e9a738589e1dd6b3cf957d1e5d8239970
 SHA512:
-  metadata.gz: 67fafa5d1998d4d1a7f9b28fdbfb20b6637e2f506eea4799a277caf3d326133d1e56a554991de05329fc44875268e5a6fb43283ec92c595bb50b98de205b166c
-  data.tar.gz: d0dbc285d27a26120887795803c35f250f415ecc1b9d8fd8a1937be313c1767f4b18d31e987a82c944c7374ea21f67cc218f01b7198c74edf344fc4cd3e60b03
+  metadata.gz: 62d898e7c52b0f89a0cf204c47503c98ddae8b210b8ea8a5cfd24f27ebb7d7940f0a0296b5a5299ffbc7bec3ef176e43c95c8128e8fb6167eb1e3b99969d7d6d
+  data.tar.gz: e02ef0697b640053766319bf917d974cd648baf64f6b50558a1b89f1c275671c31031502db460e1274d6c93ee4b682d289db5a28389ea44c183d60e6e23889c1

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.424.0)
+    rest-ftp-daemon (0.424.2)
       activesupport (~> 4.2)
       api-auth
       aws-sdk-resources (~> 2)

data/lib/rest-ftp-daemon/api/jobs.rb

@@ -5,6 +5,26 @@ module RestFtpDaemon
     class Jobs < Grape::API
       include BmcDaemonLib
 
+
+      ### EXCEPTIONS HANDLERS
+      rescue_from RestFtpDaemon::JobNotFound do |exception|
+        exception_error :api_job_not_found, 404, exception
+      end
+      rescue_from JSON::ParserError do |exception|
+        exception_error :api_job_not_found, 404, exception
+
+        log_error "JSON::ParserError: #{exception.message}"
+        error!({error: :api_parse_error, message: exception.message}, 422)
+      end
+      rescue_from RestFtpDaemon::QueueCantCreateJob do |exception|
+        exception_error :api_cant_create_job, 422, exception
+        error!({error: :api_cant_create_job, message: exception.message}, 422)
+      end
+      rescue_from RestFtpDaemonException do |exception|
+        exception_error exception_to_error(exception), 500, exception
+        Rollbar.error exception
+      end
+
       ### ENDPOINTS
       desc "Read job with ID", http_codes: [
         { code: 200, message: "Here is the job you requested" },
@@ -15,27 +35,15 @@ module RestFtpDaemon
         requires :id, type: String, desc: "ID of the Job to read"
       end
       get "/:id", requirements: { id: /.*/ } do
-        begin
-          # Get job to display
-          raise RestFtpDaemon::JobNotFound if params[:id].nil?
-          job = RestFtpDaemon::JobQueue.instance.find_by_id(params[:id]) || RestFtpDaemon::JobQueue.instance.find_by_id(params[:id], true)
-          raise RestFtpDaemon::JobNotFound if job.nil?
-
-          log_debug "found job: #{job.inspect}"
-
-        rescue RestFtpDaemon::JobNotFound => exception
-          log_error "JobNotFound: #{exception.message}"
-          error!({ error: :api_job_not_found, message: exception.message }, 404)
-
-        rescue StandardError => exception
-          log_error "Exception: #{exception.message}"
-          error!({ error: :api_exception, message: exception.message }, 500)
-
-        else
-          status 200
-          present job, with: RestFtpDaemon::Entities::Job, type: "complete"
-
-        end
+        # Get job to display
+        raise RestFtpDaemon::JobNotFound if params[:id].nil?
+        job = RestFtpDaemon::JobQueue.instance.find_by_id(params[:id]) || RestFtpDaemon::JobQueue.instance.find_by_id(params[:id], true)
+        raise RestFtpDaemon::JobNotFound if job.nil?
+        log_debug "found job: #{job.inspect}"
+
+        # Prepare response
+        status 200
+        present job, with: RestFtpDaemon::Entities::Job, type: "complete"
       end
 
       desc "List all Jobs", http_codes: [
@@ -43,19 +51,12 @@ module RestFtpDaemon
         ],
         is_array: true
       get "/" do
-        begin
-          # Get jobs to display
-          jobs = RestFtpDaemon::JobQueue.instance.jobs
-
-        rescue StandardError => exception
-          log_error "Exception: #{exception.message}"
-          error!({ error: :api_exception, message: exception.message }, 500)
-
-        else
-          status 200
-          present jobs, with: RestFtpDaemon::Entities::Job
+        # Get jobs to display
+        jobs = RestFtpDaemon::JobQueue.instance.jobs
 
-        end
+        # Prepare response
+        status 200
+        present jobs, with: RestFtpDaemon::Entities::Job
       end
 
       desc "Create a new job"
@@ -119,6 +120,21 @@ module RestFtpDaemon
           desc: "video: custom options passed to FFMPEG encoder",
           default: {}
 
+        optional :options, type: Hash, desc: "Options for transfers" do
+          optional :overwrite,
+            type: Boolean,
+            desc: "Overwrites files at target server",
+            default: Conf.at(:transfer, :overwrite)
+          optional :mkdir,
+            type: Boolean,
+            desc: "Create missing directories on target server",
+            default: Conf.at(:transfer, :mkdir)
+          optional :tempfile,
+            type: Boolean,
+            desc: "Upload to a temp file before renaming it to the target filename",
+            default: Conf.at(:transfer, :tempfile)
+        end
+
         optional :overwrite,
           type: Boolean,
           desc: "Overwrites files at target server",
@@ -140,31 +156,15 @@ module RestFtpDaemon
       end
 
       post "/" do
-        # log_debug params.to_json
-        begin
-          # Add up the new job on the queue
-          job = RestFtpDaemon::JobQueue.instance.create_job(params)
+        # Add up the new job on the queue
+        job = RestFtpDaemon::JobQueue.instance.create_job(params)
 
-          # Increment a counter
-          RestFtpDaemon::Counters.instance.increment :jobs, :received
+        # Increment a counter
+        RestFtpDaemon::Counters.instance.increment :jobs, :received
 
-        rescue JSON::ParserError => exception
-          log_error "JSON::ParserError: #{exception.message}"
-          error!({error: :api_parse_error, message: exception.message}, 422)
-
-        rescue QueueCantCreateJob => exception
-          log_error "QueueCantCreateJob: #{exception.message}"
-          error!({error: :api_cant_create_job, message: exception.message}, 422)
-
-        rescue RestFtpDaemonException => exception
-          log_error "#{exception.class.to_s} #{exception.message}"
-          error!({error: exception_to_error(exception), message: exception.message}, 500)
-
-        else
-          status 201
-          present job, with: RestFtpDaemon::Entities::Job, hide_params: true
-
-        end
+        # Prepare response
+        status 201
+        present job, with: RestFtpDaemon::Entities::Job, hide_params: true
       end
 
     end

data/lib/rest-ftp-daemon/api/root.rb

@@ -22,12 +22,37 @@ module RestFtpDaemon
         def logger
           Root.logger
         end
+
+        def exception_error name, http_code, exception
+          # Extract message lines
+          lines = exception.message.lines.collect(&:strip).reject(&:empty?)
+
+          # Log error to file
+          log_error "#{http_code} [#{name}] #{lines.shift} ", lines
+
+          # Return error
+          error!({
+            error: name,
+            http_code: http_code,
+            message: exception.message
+          }, http_code)
+        end
+
       end
 
       before do
         log_request
       end
 
+
+      ## EXCEPTION HANDLERS
+      rescue_from :all do |exception|
+        Rollbar.error exception
+        #error!({error: :internal_server_error, message: exception.message}, 500)
+        exception_error :internal_server_error, 500, exception
+      end
+
+
       ### CLASS CONFIG
       logger BmcDaemonLib::LoggerPool.instance.get :api
       do_not_route_head!
@@ -50,12 +75,6 @@ module RestFtpDaemon
         put "-----"
       end
 
-      ## GLOBAL EXCEPTION HANDLING
-      rescue_from :all do |exception|
-        Rollbar.error exception
-        error_response(message: "Internal server error: #{exception}", status: 500)
-      end
-
       ### MOUNTPOINTS
       mount RestFtpDaemon::API::Status      => MOUNT_STATUS
       mount RestFtpDaemon::API::Jobs        => MOUNT_JOBS

data/lib/rest-ftp-daemon/constants.rb

@@ -16,8 +16,8 @@ JOB_FFMPEG_THREADS      = 2
 JOB_FFMPEG_ATTRIBUTES   = [:video_codec, :video_bitrate, :video_bitrate_tolerance, :frame_rate, :resolution, :aspect, :keyframe_interval, :x264_vprofile, :x264_preset, :audio_codec, :audio_bitrate, :audio_sample_rate, :audio_channels]
 
 # Internal job infos
-INFO_PROGRESS             = :work_progress
-INFO_BITRATE              = :transfer_bitrate
+INFO_PROGRESS             = :progress
+INFO_BITRATE              = :bitrate
 
 # Constants: logger
 LOG_FORMAT_PROGNAME      = "%d\t%s"

data/lib/rest-ftp-daemon/job.rb

@@ -15,7 +15,7 @@ module RestFtpDaemon
     include BmcDaemonLib::LoggerHelper
 
     # Fields to be imported from params
-    IMPORTED = %w(type priority pool label priority source target overwrite mkdir tempfile video_options video_custom)
+    IMPORTED = %w(type priority pool label priority source target overwrite notify mkdir tempfile video_options video_custom)
 
     # Class options
     attr_accessor :wid

data/lib/rest-ftp-daemon/jobs/transfer.rb

@@ -145,63 +145,79 @@ module RestFtpDaemon
 
       # Compute final bitrate
       global_transfer_bitrate = get_bitrate @transfer_total, (Time.now - transfer_started_at)
-      set_info :transfer_bitrate, global_transfer_bitrate.round(0)
+      set_info INFO_BITRATE, global_transfer_bitrate.round(0)
 
       # Done
       set_info :source_current, nil
     end
 
     def update_progress transferred, name = ""
-      # What's current time ?
-      now = Time.now
-      notify_after = @config[:notify_after]
-
       # Update counters
       @transfer_sent += transferred
       set_info :work_sent, @transfer_sent
 
       # Update job info
       percent0 = (100.0 * @transfer_sent / @transfer_total).round(0)
-      set_info INFO_PROGRESS, percent0
-
-      # Update job status after each NOTIFY_UPADE_STATUS
-      progressed_ago = (now.to_f - @progress_at.to_f)
-      if (!JOB_UPDATE_INTERVAL.to_f.zero?) && (progressed_ago > JOB_UPDATE_INTERVAL.to_f)
-        @current_bitrate = running_bitrate @transfer_sent
-        set_info :transfer_bitrate, @current_bitrate.round(0)
-
-        # Log progress
-        stack = []
-        stack << "#{percent0} %"
-        stack << (format_bytes @transfer_sent, "B")
-        stack << (format_bytes @transfer_total, "B")
-        stack << (format_bytes @current_bitrate.round(0), "bps")
-        stack2 = stack.map { |txt| ("%#{LOG_PIPE_LEN.to_i}s" % txt) }.join("\t")
-        log_debug "progress #{stack2} \t#{name}"
-
-        # Remember when we last did it
-        @progress_at = now
-      end
+      set_info INFO_PROGRESS,  percent0
+
+      # What's current time ?
+      now = Time.now
+
+      # Update job status
+      update_progress_jobinfo now, percent0, name
 
       # Notify if requested
-      notified_ago = (now.to_f - @notified_at.to_f)
-      if (!notify_after.nil?) && (notified_ago > notify_after)
-        # Prepare and send notification
-        notif_status = {
-          progress: percent0,
-          transfer_sent: @transfer_sent,
-          transfer_total: @transfer_total,
-          transfer_bitrate: @current_bitrate.round(0),
-          }
-        client_notify :progress, status: notif_status
-
-        # Remember when we last did it
-        @notified_at = now
-      end
+      update_progress_notify now, percent0, name
     end
 
   private
 
+    def update_progress_jobinfo now, percent0, name
+      # No delay provided ?
+      return if JOB_UPDATE_INTERVAL.to_f.zero?
+
+      # Still too early to notify again ?
+      how_long_ago = (now.to_f - @progress_at.to_f)
+      return unless how_long_ago > JOB_UPDATE_INTERVAL.to_f
+
+      # Update bitrates
+      @current_bitrate = running_bitrate @transfer_sent
+      set_info INFO_BITRATE,   @current_bitrate.round(0)
+
+      # Log progress
+      stack = [
+        "#{percent0} %",
+        format_bytes(@transfer_sent, "B"),
+        format_bytes(@current_bitrate.round(0), "bps")
+        ]
+      stack2 = stack.map { |txt| ("%#{LOG_PIPE_LEN.to_i}s" % txt) }.join("\t")
+      log_debug "progress #{stack2} \t#{name}"
+
+      # Remember when we last did it
+      @progress_at = now
+    end
+
+    def update_progress_notify now, percent0, name
+      # No delay provided ?
+      return if @config[:notify_after].nil?
+
+      # Still too early to notify again ?
+      how_long_ago = (now.to_f - @notified_at.to_f)
+      return unless how_long_ago > @config[:notify_after]
+
+      # Prepare and send notification
+      client_notify :progress, status: {
+        progress: percent0,
+        transfer_sent: @transfer_sent,
+        transfer_total: @transfer_total,
+        transfer_bitrate: @current_bitrate.round(0),
+        transfer_current: name,
+        }
+
+      # Remember when we last did it
+      @notified_at = now
+    end
+
     def get_bitrate delta_data, delta_time
       return nil if delta_time.nil? || delta_time.zero?
       8 * delta_data.to_f.to_f / delta_time

data/lib/rest-ftp-daemon/workers/transfer.rb

@@ -35,37 +35,14 @@ module RestFtpDaemon
       job = RestFtpDaemon::JobQueue.instance.pop @pool
 
       # Work on this job
-      work_on_job(job)
+      work_on_job job
 
       # Clean job status
       job.wid = nil
       #sleep 1
 
-      # If job status requires a retry, just restack it
-      if !job.error
-        #log_info "job succeeded"
-
-      elsif !(@config[:retry_on].is_a?(Enumerable) && @config[:retry_on].include?(job.error))
-        log_error "not retrying: error not eligible"
-
-      elsif @config[:retry_for] && (job.age >= @config[:retry_for])
-        log_error "not retrying: max_age reached (#{@config[:retry_for]} s)"
-
-      elsif @config[:retry_max] && (job.runs >= @config[:retry_max])
-        log_error "not retrying: max_runs reached (#{@config[:retry_max]} tries)"
-
-      else
-        # Delay cannot be negative, and will be 1s minimum
-        retry_after = [@config[:retry_after] || DEFAULT_RETRY_AFTER, 1].max
-        log_info "retrying job: waiting for #{retry_after} seconds"
-
-        # Wait !
-        sleep retry_after
-        log_info "retrying job: requeued after delay"
-
-        # Now, requeue this job
-        RestFtpDaemon::JobQueue.instance.requeue job
-      end
+      # Handle the retry if needed
+      handle_job_result job
     end
 
     def work_on_job job
@@ -92,26 +69,40 @@ module RestFtpDaemon
       # Inform the job
       job.oops_you_stop_now ex unless job.nil?
 
-    rescue RestFtpDaemon::AttributeMissing => ex
-      log_error "JOB MISSING ATTRIBUTE", ex.backtrace
+    rescue RestFtpDaemon::AssertionFailed, RestFtpDaemon::AttributeMissing, StandardError => ex
+      log_error "JOB EXCEPTION ex[#{ex.class}] #{ex.message}", ex.backtrace
       worker_status WORKER_STATUS_CRASHED
 
       # Inform the job
-      job.oops_you_stop_now ex unless job.nil?
+      job.oops_after_crash ex unless job.nil?
+    end
 
-    rescue RestFtpDaemon::AssertionFailed => ex
-      log_error "JOB ASSERTION FAILED", ex.backtrace
-      worker_status WORKER_STATUS_CRASHED
+    def handle_job_result job
+      # If job status requires a retry, just restack it
+      if !job.error
+        #log_info "job succeeded"
 
-      # Inform the job
-      job.oops_you_stop_now ex unless job.nil?
+      elsif !(@config[:retry_on].is_a?(Enumerable) && @config[:retry_on].include?(job.error))
+        log_error "not retrying: error not eligible"
 
-    rescue StandardError => ex
-      log_error "JOB UNHANDLED EXCEPTION ex[#{ex.class}] #{ex.message}", ex.backtrace
-      worker_status WORKER_STATUS_CRASHED
+      elsif @config[:retry_for] && (job.age >= @config[:retry_for])
+        log_error "not retrying: max_age reached (#{@config[:retry_for]} s)"
 
-      # Inform the job
-      job.oops_after_crash ex unless job.nil?
+      elsif @config[:retry_max] && (job.runs >= @config[:retry_max])
+        log_error "not retrying: max_runs reached (#{@config[:retry_max]} tries)"
+
+      else
+        # Delay cannot be negative, and will be 1s minimum
+        retry_after = [@config[:retry_after] || DEFAULT_RETRY_AFTER, 1].max
+        log_info "retrying job: waiting for #{retry_after} seconds"
+
+        # Wait !
+        sleep retry_after
+        log_info "retrying job: requeued after delay"
+
+        # Now, requeue this job
+        RestFtpDaemon::JobQueue.instance.requeue job
+      end
     end
 
   end

data/rest-ftp-daemon.gemspec

@@ -2,7 +2,7 @@
 Gem::Specification.new do |spec|
 
   # Project version
-  spec.version                    = "0.424.0"
+  spec.version                    = "0.424.2"
 
   # Project description
   spec.name                       = "rest-ftp-daemon"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: 0.424.0
+  version: 0.424.2
 platform: ruby
 authors:
 - Bruno MEDICI
@@ -414,8 +414,6 @@ files:
 - lib/rest-ftp-daemon/api/dashboard.rb
 - lib/rest-ftp-daemon/api/debug.rb
 - lib/rest-ftp-daemon/api/jobs.rb
-- lib/rest-ftp-daemon/api/root-real.rb
-- lib/rest-ftp-daemon/api/root-test.rb
 - lib/rest-ftp-daemon/api/root.rb
 - lib/rest-ftp-daemon/api/status.rb
 - lib/rest-ftp-daemon/constants.rb

data/lib/rest-ftp-daemon/api/root-real.rb

@@ -1,80 +0,0 @@
-require "grape"
-require 'grape-swagger'
-
-module RestFtpDaemon
-  module API
-    class Root < Grape::API
-      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
-
-      ### LOGGING & HELPERS
-      helpers RestFtpDaemon::CommonHelpers
-      helpers RestFtpDaemon::ApiHelpers
-      helpers BmcDaemonLib::LoggerHelper
-
-      helpers do
-        def log_prefix
-          ['API', nil, nil]
-        end
-
-        def logger
-          Root.logger
-        end
-      end
-
-      before do
-        log_request
-      end
-
-      ### CLASS CONFIG
-      logger BmcDaemonLib::LoggerPool.instance.get :api
-      do_not_route_head!
-      do_not_route_options!
-      # version 'v1'
-      format :json
-      content_type :json, 'application/json; charset=utf-8'
-
-
-desc 'API Root'
-
-      ### MOUNTPOINTS
-      # mount RestFtpDaemon::API::Status      => MOUNT_STATUS
-      # mount RestFtpDaemon::API::Jobs        => MOUNT_JOBS
-      # mount RestFtpDaemon::API::Dashbaord   => MOUNT_BOARD
-      # mount RestFtpDaemon::API::Config      => MOUNT_CONFIG
-      # mount RestFtpDaemon::API::Debug       => MOUNT_DEBUG
-
-
-      ### API Documentation
-      add_swagger_documentation hide_documentation_path: true,
-        api_version: BmcDaemonLib::Conf.app_ver,
-        doc_version: BmcDaemonLib::Conf.app_ver,
-        mount_path: MOUNT_SWAGGER_JSON,
-        info: {
-          title: BmcDaemonLib::Conf.app_name,
-          version: BmcDaemonLib::Conf.app_ver,
-          description: "API description for #{BmcDaemonLib::Conf.app_name} #{BmcDaemonLib::Conf.app_ver}",
-          }
-         # models: [
-         #   RestFtpDaemon::Entities::Job,
-         # ]
-
-      ### GLOBAL EXCEPTION HANDLING
-      # rescue_from :all do |e|
-      #   raise e
-      #   error_response(message: "Internal server error: #{e}", status: 500)
-      # end
-
-
-      ### INITIALIZATION
-      def initialize
-        super
-      end
-
-      ### ENDPOINTS
-      get "/" do
-        redirect dashboard_url()
-      end
-
-    end
-  end
-end

data/lib/rest-ftp-daemon/api/root-test.rb

@@ -1,69 +0,0 @@
-require "grape"
-require 'grape-swagger'
-
-module RestFtpDaemon
-  module API
-    class Root < Grape::API
-
-      ### API Documentation
-      add_swagger_documentation hide_documentation_path: true,
-        api_version: Conf.app_ver,
-        doc_version: Conf.app_ver,
-        mount_path: MOUNT_SWAGGER_JSON,
-        info: {
-          title: Conf.app_name,
-          version: Conf.app_ver,
-          description: "API description for #{Conf.app_name} #{Conf.app_ver}",
-          }
-
-
-      namespace :hudson do
-        desc 'Document root'
-        get '/' do
-        end
-      end
-
-      namespace :hudson do
-        desc 'This gets something.',
-          notes: '_test_'
-
-        get '/simple' do
-          { bla: 'something' }
-        end
-      end
-
-      namespace :colorado do
-        desc 'This gets something for URL using - separator.',
-          notes: '_test_'
-
-        get '/simple-test' do
-          { bla: 'something' }
-        end
-
-
-      desc "List all Jobs", http_codes: [
-        { code: 200, message: "Here are the jobs you requested" },
-        ],
-        is_array: true
-      get "/" do
-        begin
-          # Get jobs to display
-          jobs = RestFtpDaemon::JobQueue.instance.jobs
-
-        rescue StandardError => exception
-          log_error "Exception: #{exception.message}"
-          error!({ error: :api_exception, message: exception.message }, 500)
-
-        else
-          status 200
-          present jobs, with: RestFtpDaemon::Entities::Job
-
-        end
-      end
-
-
-      end
-
-    end
-  end
-end