rest-ftp-daemon 0.435.2 → 0.501.0

data/lib/rest-ftp-daemon/counters.rb

@@ -43,4 +43,4 @@ module RestFtpDaemon
     end
 
   end
-end
+end

data/lib/rest-ftp-daemon/entities/job.rb

@@ -50,10 +50,6 @@ module RestFtpDaemon
       # Source and target     #, :unless => Proc.new {|g| g.source_loc.nil?}
       expose :source_loc, using: Entities::Location#, as: :source
       expose :target_loc, using: Entities::Location#, as: :target
-
-      # expose :slots do |station,options|
-      #   station.slots.map{ |slot| SlotEntity.new(slot).serializable_hash }
-      # end
     end
   end
-end
+end

data/lib/rest-ftp-daemon/entities/location.rb

@@ -4,8 +4,9 @@ module RestFtpDaemon
   module Entities
     class Location < Grape::Entity
 
-      expose :original, as: '_'
-      expose :uri
+      expose :url
+      #, as: 'raw'
+      # expose :uri
       expose :scheme
 
       expose :host, unless: Proc.new {|obj| obj.host.nil?}
@@ -22,4 +23,4 @@ module RestFtpDaemon
 
     end
   end
-end
+end

data/lib/rest-ftp-daemon/entities/options.rb

@@ -10,4 +10,4 @@ module RestFtpDaemon
 
     end
   end
-end
+end

data/lib/rest-ftp-daemon/exceptions.rb

@@ -28,4 +28,4 @@ module RestFtpDaemon
   class VideoMovieError           < BaseException; end
 
 
-end
+end

data/lib/rest-ftp-daemon/helpers/api.rb

@@ -22,4 +22,4 @@ module RestFtpDaemon
     end
 
   end
-end
+end

data/lib/rest-ftp-daemon/helpers/common.rb

@@ -34,4 +34,4 @@ module RestFtpDaemon
     end
 
   end
-end
+end

data/lib/rest-ftp-daemon/helpers/views.rb

@@ -40,6 +40,8 @@ module RestFtpDaemon
         "success"
       when URI::S3
         "primary"
+      when URI::Generic
+        "info"
       else
         "default"
       end
@@ -58,15 +60,6 @@ module RestFtpDaemon
       end
     end
 
-    def location_label uri
-      sprintf(
-        '<div class="transfer-type label label-%s" title="%s">%s</div>',
-        location_style(uri),
-        uri.to_s,
-        uri.class.name.split('::').last
-        )
-    end
-
     def job_type job
       # sprintf(
       #     '<span class="glyphicon glyphicon-%s" alt="%s"></span>&nbsp;%s',
@@ -118,6 +111,32 @@ module RestFtpDaemon
       path.gsub(/\[([^\[]+)\]/, token_to_label('\1'))
     end
 
+    def location_label loc
+      # sprintf(
+      #   '<div class="transfer-type label label-%s" title="%s">%s</div>',
+      #   location_style(loc.uri),
+      #   loc.to_s,
+      #   loc.uri.class.name.split('::').last
+      #   )
+      sprintf(
+        '
+        <span class="label-group">
+          <span class="transfer-type label label-xs label-%s" title="%s">%s</span><span class="label label-simple" title="%s">%s</span>
+        </span>
+        ',
+        location_style(loc.uri),
+        loc.to_s,
+        loc.uri.class.name.split('::').last,
+        loc.tokens.first,
+        loc.tokens.first,
+        )
+    end
+
+    # def token_highlight path
+    #   return unless path.is_a? String
+    #   path.gsub(/\[([^\[]+)\]/, token_to_label('\1'))
+    # end
+
     def text_or_empty text
       return "-" if text.nil? || text.empty?
       text
@@ -125,4 +144,4 @@ module RestFtpDaemon
 
 
   end
-end
+end

data/lib/rest-ftp-daemon/initialize.rb

@@ -11,4 +11,4 @@ end
 
 # Initialize workers
 Conf.log :initialize, "prepare workers"
-RestFtpDaemon::WorkerPool.instance.start_em_all
+RestFtpDaemon::WorkerPool.instance.start_em_all

data/lib/rest-ftp-daemon/job.rb

@@ -85,7 +85,7 @@ module RestFtpDaemon
       @target_loc = Location.new(params[:target])
 
       # We're done!
-      log_info "Job.initialized", {
+      log_info "initialized", {
         source: @source_loc.uri,
         target: @target_loc.uri,
         pool: @pool,
@@ -110,7 +110,7 @@ module RestFtpDaemon
       set_status JOB_STATUS_QUEUED
       set_error nil
       client_notify :queued
-      log_info "Job.reset notify[queued] tentative[#{@tentatives}]"
+      log_info "reset notify[queued] tentative[#{@tentatives}]"
     end
 
     # Process job
@@ -123,23 +123,23 @@ module RestFtpDaemon
       @started_at = Time.now
 
       # Notify we start working
-      log_info "Job.start notify [started]"
+      log_info "client_notify [started]"
       current_signal = :started
       set_status JOB_STATUS_WORKING
       client_notify :started
 
       # Before work
-      log_debug "Job.start do_before"
+      log_debug "do_before"
       current_signal = :started
       do_before
 
       # Do the hard work
-      log_debug "Job.start do_work"
+      log_debug "do_work"
       current_signal = :ended
       do_work
 
       # Finalize all this
-      log_debug "Job.start do_after"
+      log_debug "do_after"
       current_signal = :ended
       do_after
 
@@ -150,7 +150,7 @@ module RestFtpDaemon
     else
       # All done !
       set_status JOB_STATUS_FINISHED
-      log_info "Job.start notify [ended]"
+      log_info "client_notify [ended]"
       client_notify :ended
     end
 
@@ -217,7 +217,7 @@ module RestFtpDaemon
   protected
 
     def alert_common_method_called
-      log_error "Job PLACEHOLDER METHOD CALLED"
+      log_error "PLACEHOLDER METHOD CALLED"
     end
 
   private
@@ -289,25 +289,23 @@ module RestFtpDaemon
       RestFtpDaemon::Notification.new @notify, payload
 
     rescue StandardError => ex
-      log_error "Job.client_notify EXCEPTION: #{ex.inspect}"
+      log_error "client_notify EXCEPTION: #{ex.inspect}"
     end
 
     def oops signal, exception, error = nil#, include_backtrace = false
       # Find error code in ERRORS table
       if error.nil?
         error = ERRORS.key(exception.class)
-        # log_debug "Job.oops ERRORS: #{exception.class} > #{error}"
       end
 
       # Default error code derived from exception name
       if error.nil?
         error = exception_to_error(exception)
-        # log_debug "Job.oops derivated: #{exception.class} > #{error}"
         include_backtrace = true
       end
 
       # Log backtrace ?
-      message = "Job.oops signal[#{signal}] exception[#{exception.class}] error[#{error}] #{exception.message}"
+      message = "oops signal[#{signal}] exception[#{exception.class}] error[#{error}] #{exception.message}"
       if include_backtrace
         log_error message, exception.backtrace
       else
@@ -349,4 +347,4 @@ module RestFtpDaemon
     add_transaction_tracer :initialize,     category: :task
 
   end
-end
+end

data/lib/rest-ftp-daemon/job_queue.rb

@@ -30,7 +30,7 @@ module RestFtpDaemon
 
       # Identifiers generator
       @prefix = identifier(JOB_IDENT_LEN)
-      log_info "JobQueue initialized (prefix: #{@prefix})"
+      log_info "initialized (prefix: #{@prefix})"
     end
 
     def create_job params
@@ -42,7 +42,7 @@ module RestFtpDaemon
       # If object not found, don't create a job !
       unless klass && klass < Job
         message = "can't create [#{klass_name}] for type [#{params[:type]}]"
-        log_error "JobQueue.create_job: #{message}"
+        log_error "create_job: #{message}"
         raise QueueCantCreateJob, message
       end
 
@@ -53,7 +53,7 @@ module RestFtpDaemon
       job_id = prefixed_id(@last_id)
 
       # Instantiate it and return the now object
-      log_info "JobQueue.create_job: creating [#{klass.name}] with ID [#{job_id}]"
+      log_info "create_job: creating [#{klass.name}] with ID [#{job_id}]"
       job = klass.new(job_id, params)
 
       # Push it on the queue
@@ -147,10 +147,10 @@ module RestFtpDaemon
 
     def push job
       # Check that item responds to "priorty" method
-      raise "JobQueue.push: job should respond to: priority" unless job.respond_to? :priority
-      raise "JobQueue.push: job should respond to: id" unless job.respond_to? :id
-      raise "JobQueue.push: job should respond to: pool" unless job.respond_to? :pool
-      raise "JobQueue.push: job should respond to: reset" unless job.respond_to? :reset
+      raise "push: job should respond to: priority" unless job.respond_to? :priority
+      raise "push: job should respond to: id" unless job.respond_to? :id
+      raise "push: job should respond to: pool" unless job.respond_to? :pool
+      raise "push: job should respond to: reset" unless job.respond_to? :reset
 
       @mutex.synchronize do
         # Get this job's pool & prepare queue of this pool
@@ -190,7 +190,6 @@ module RestFtpDaemon
         @waitings[pool] ||= []
         loop do
           if myqueue.empty?
-            #puts "JobQueue.pop(#{pool}): empty"
             raise ThreadError, "queue empty" if non_block
             @waitings[pool].push Thread.current
             @mutex.sleep
@@ -228,7 +227,7 @@ module RestFtpDaemon
 
       # Compute oldest limit
       time_limit = Time.now - maxage.to_i
-      log_info "JobQueue.expire limit [#{time_limit}] status [#{status}]" if verbose
+      log_info "expire limit [#{time_limit}] status [#{status}]" if verbose
 
       @mutex.synchronize do
         # Delete jobs from the queue when they match status and age limits
@@ -269,4 +268,4 @@ module RestFtpDaemon
     add_transaction_tracer :jobs_by_status,       category: :task
 
   end
-end
+end

data/lib/rest-ftp-daemon/jobs/dummy.rb

@@ -17,4 +17,4 @@ module RestFtpDaemon
     end
 
   end
-end
+end

data/lib/rest-ftp-daemon/jobs/errors.rb

@@ -5,12 +5,16 @@ require "net/ftp"
 require 'streamio-ffmpeg'
 
 module RestFtpDaemon
+  class InvalidWorkerNumber       < BaseException; end
+  class QueueCantCreateJob        < BaseException; end
+  class JobException              < BaseException; end
+  class JobNotFound               < BaseException; end
   class Job
 
     # Common errors
     ERRORS = {
-      invalid_argument:         Errno::EINVAL,
-      runtime_error:            RuntimeError,
+      # oops_invalid_argument:         Errno::EINVAL,
+      oops_runtime_error:            RuntimeError,
 
       job_timeout:              RestFtpDaemon::JobTimeout,
       source_not_supported:     RestFtpDaemon::SourceUnsupported,
@@ -41,7 +45,10 @@ module RestFtpDaemon
       ftp_proto_error:          Net::FTPProtoError,
       ftp_error:                Net::FTPError,
 
-      ffmpeg_error:             FFMPEG::Error,
+      sftp_exception:           Net::SFTP::StatusException,
+      sftp_key_mismatch:        Net::SSH::HostKeyMismatch,
+      sftp_auth_failed:         Net::SSH::AuthenticationFailed,
+      sftp_openssl_error:       OpenSSL::SSL::SSLError,
 
       s3_no_such_waiter:        Aws::Waiters::Errors::NoSuchWaiterError,
       s3_failure_state_error:   Aws::Waiters::Errors::FailureStateError,
@@ -49,30 +56,21 @@ module RestFtpDaemon
       s3_waiter_unexpected:     Aws::Waiters::Errors::UnexpectedError,
       s3_waiter_failed:         Aws::Waiters::Errors::WaiterFailed,
 
+      #s3_not_found:             Aws::S3::Errors::NotFound,
       s3_permanent_redirect:    Aws::S3::Errors::PermanentRedirect,
       s3_no_such_key:           Aws::S3::Errors::NoSuchKey,
       s3_no_such_bucket:        Aws::S3::Errors::NoSuchBucket,
       s3_no_such_upload:        Aws::S3::Errors::NoSuchUpload,
       s3_error:                 Aws::S3::Errors::ServiceError,
 
-      sftp_exception:           Net::SFTP::StatusException,
-      sftp_key_mismatch:        Net::SSH::HostKeyMismatch,
-      sftp_auth_failed:         Net::SSH::AuthenticationFailed,
-      sftp_openssl_error:       OpenSSL::SSL::SSLError,
-
       video_missing_binary:     RestFtpDaemon::VideoMissingBinary,
       video_movie_error:        RestFtpDaemon::VideoMovieError,
+      video_ffmpeg_error:             FFMPEG::Error,
 
       # rescue Encoding::UndefinedConversionError => exception
       #   return oops :ended, exception, "encoding_error", true
       }
 
-  class InvalidWorkerNumber       < BaseException; end
-  class QueueCantCreateJob        < BaseException; end
-  class JobException              < BaseException; end
-  class JobNotFound               < BaseException; end
-
-
 
   end
-end
+end

data/lib/rest-ftp-daemon/jobs/transfer.rb

@@ -21,20 +21,20 @@ module RestFtpDaemon
       # Prepare remote object
       case target_uri
       when URI::FTP
-        log_info "JobTransfer.do_before target_method FTP"
+        log_info "do_before target_method FTP"
         @remote = Remote::RemoteFTP.new @target_loc, log_context, @config[:debug_ftp]
       when URI::FTPES, URI::FTPS
-        log_info "JobTransfer.do_before target_method FTPES/FTPS"
+        log_info "do_before target_method FTPES/FTPS"
         @remote = Remote::RemoteFTP.new @target_loc, log_context, @config[:debug_ftps], :ftpes
       when URI::SFTP
-        log_info "JobTransfer.do_before target_method SFTP"
+        log_info "do_before target_method SFTP"
         @remote = Remote::RemoteSFTP.new @target_loc, log_context, @config[:debug_sftp]
       when URI::S3
-        log_info "JobTransfer.do_before target_method S3"
+        log_info "do_before target_method S3"
         @remote = Remote::RemoteS3.new @target_loc, log_context, @config[:debug_s3]
       else
         message = "unknown scheme [#{@target_loc.scheme}] [#{target_uri.class.name}]"
-        log_info "JobTransfer.do_before #{message}"
+        log_info "do_before #{message}"
         raise RestFtpDaemon::TargetUnsupported, message
       end
 
@@ -48,10 +48,10 @@ module RestFtpDaemon
     def do_work
       # Scan local source files from disk
       set_status JOB_STATUS_CHECKING_SRC
-      sources = @source_loc.scan_files
+      sources = @source_loc.local_files
       set_info INFO_SOURCE_COUNT, sources.size
       set_info INFO_SOURCE_FILES, sources.collect(&:name)
-      log_info "JobTransfer.do_work sources #{sources.collect(&:name)}"
+      log_info "do_work sources #{sources.collect(&:name)}"
       raise RestFtpDaemon::SourceNotFound if sources.empty?
 
       # Guess target file name, and fail if present while we matched multiple sources
@@ -63,7 +63,8 @@ module RestFtpDaemon
 
       # Prepare target path or build it if asked
       set_status JOB_STATUS_CHDIR
-      @remote.chdir_or_create @target_loc.dir, @mkdir
+      #log_info "do_work chdir_or_create #{@target_loc.filedir}"
+      @remote.chdir_or_create @target_loc.filedir, @mkdir
 
       # Compute total files size
       @transfer_total = sources.collect(&:size).sum
@@ -95,7 +96,7 @@ module RestFtpDaemon
 
     def do_after
       # Close FTP connexion and free up memory
-      log_info "JobTransfer.do_after close connexion, update status and counters"
+      log_info "do_after close connexion, update status and counters"
       @remote.close
 
       # Free @remote object
@@ -117,14 +118,14 @@ module RestFtpDaemon
       raise RestFtpDaemon::AssertionFailed, "remote_upload/target" if target.nil?
 
       # Use source filename if target path provided none (typically with multiple sources)
-      log_info "JobTransfer.remote_upload [#{source.name}]: [#{source.path}] > [#{target.path}]"
+      log_info "remote_upload temp[#{@tempfile}] source[#{source.path}] target[#{target.path}]"
       set_info INFO_SOURCE_CURRENT, source.name
 
       # Remove any existing version if present, or check if it's there
       if @overwrite
         @remote.remove! target
-      elsif size = @remote.present?(target)
-        log_debug "JobTransfer.remote_upload existing (#{format_bytes size, 'B'})"
+      elsif (size = @remote.size_if_exists(target))  # won't be triggered when NIL or 0 is returned
+        log_debug "remote_upload file exists ! (#{format_bytes size, 'B'})"
         raise RestFtpDaemon::TargetFileExists
       end
 
@@ -134,7 +135,6 @@ module RestFtpDaemon
 
       # Start the transfer, update job status after each block transfer
       set_status JOB_STATUS_UPLOADING
-      log_debug "JobTransfer.remote_upload source[#{source.path}] temp[#{@tempfile}]"
       @remote.upload source, target, @tempfile do |transferred, name|
         # Update transfer statistics
         update_progress transferred, name
@@ -192,7 +192,7 @@ module RestFtpDaemon
         format_bytes(@current_bitrate.round(0), "bps")
         ]
       stack2 = stack.map { |txt| ("%#{LOG_PIPE_LEN.to_i}s" % txt) }.join("\t")
-      log_debug "progress #{stack2} \t#{name}"
+      log_info "progress #{stack2} \t#{name}"
 
       # Prepare and send notification
       client_notify :progress, status: {
@@ -233,4 +233,4 @@ module RestFtpDaemon
   # add_transaction_tracer :prepare,        category: :task
   # add_transaction_tracer :run,            category: :task
 
-end
+end