rest-ftp-daemon 0.400.0 → 0.410.0.pre.1

data/lib/rest-ftp-daemon/job_queue.rb

@@ -35,6 +35,35 @@ module RestFtpDaemon
       log_info "JobQueue initialized (prefix: #{@prefix})"
     end
 
+    def create_job params
+      # Build class name and chock if it exists
+      # klass = Kernel.const_get("Job#{params[:type].to_s.capitalize}") rescue nil
+      klass_name = "Job#{params[:type].to_s.capitalize}"
+      klass = Kernel.const_get(klass_name) rescue nil
+
+      # If object not found, don't create a job !
+      unless klass && klass < Job
+        message = "can't create [#{klass_name}] for type [#{params[:type]}]"
+        log_error "JobQueue.create_job: #{message}"
+        raise QueueCantCreateJob, message
+      end
+
+      # Generate an ID and stack it
+      @mutex.synchronize do
+        @last_id += 1
+      end
+      job_id = prefixed_id(@last_id)
+
+      # Instantiate it and return the now object
+      log_info "JobQueue.create_job: creating [#{klass.name}] with ID [#{job_id}]"
+      job = klass.new(job_id, params)
+
+      # Push it on the queue
+      push job
+
+      return job
+    end
+
     def generate_id
       @mutex.synchronize do
         @last_id += 1
@@ -121,13 +150,13 @@ module RestFtpDaemon
 
     def push job
       # Check that item responds to "priorty" method
-      raise "JobQueue.push: job should respond to priority method" unless job.respond_to? :priority
-      raise "JobQueue.push: job should respond to id method" unless job.respond_to? :id
-      raise "JobQueue.push: job should respond to pool method" unless job.respond_to? :pool
-      raise "JobQueue.push: job should respond to reset" unless job.respond_to? :reset
+      raise "JobQueue.push: job should respond to: priority" unless job.respond_to? :priority
+      raise "JobQueue.push: job should respond to: id" unless job.respond_to? :id
+      raise "JobQueue.push: job should respond to: pool" unless job.respond_to? :pool
+      raise "JobQueue.push: job should respond to: reset" unless job.respond_to? :reset
 
       @mutex.synchronize do
-        # Get this job's pool @ prepare queue of this pool
+        # Get this job's pool & prepare queue of this pool
         pool = job.pool
         myqueue = (@queues[pool] ||= [])
 

data/lib/rest-ftp-daemon/jobs/dummy.rb

@@ -0,0 +1,20 @@
+module RestFtpDaemon
+  class JobDummy < Job
+
+    # def initialize job_id, params = {}
+    #   super
+    # end
+
+    def before
+    end
+
+    def work
+      log_info "JobDummy.work YEAH WE'RE PROCESSING, man !"
+      sleep 5
+    end
+
+    def after
+    end
+
+  end
+end

data/lib/rest-ftp-daemon/jobs/transfer.rb

@@ -0,0 +1,319 @@
+#FIXME: move progress from Job/infos/transfer to Job/progress
+
+module RestFtpDaemon
+  class JobTransfer < Job
+
+  protected
+
+    def before
+      # Prepare flags
+      flag_prepare :mkdir, false
+      flag_prepare :overwrite, false
+      flag_prepare :tempfile, true
+
+      # Some init
+      @transfer_sent = 0
+      set_info :source, :processed, 0
+
+      # Ensure source is FILE
+      raise RestFtpDaemon::SourceNotSupported, @source_loc.scheme   unless source_uri.is_a? URI::FILE
+
+      # Prepare remote (case would be preferable but too hard to use,
+      # as target could be of a descendent class of URI:XXX and not matching directly)
+      case target_uri
+      when URI::FTP
+        log_info "JobTransfer.before target_method FTP"
+        @remote = RemoteFTP.new @target_loc.uri, log_prefix, debug: @config[:debug_ftp]
+      when URI::FTPES, URI::FTPS
+        log_info "JobTransfer.before target_method FTPES/FTPS"
+        @remote = RemoteFTP.new @target_loc.uri, log_prefix, debug: @config[:debug_ftps], ftpes: true
+      when URI::SFTP
+        log_info "JobTransfer.before target_method SFTP"
+        @remote = RemoteSFTP.new @target_loc.uri, log_prefix, debug: @config[:debug_sftp]
+      else
+        log_info "JobTransfer.before unknown scheme [#{@target_loc.scheme}]"
+        raise RestFtpDaemon::TargetNotSupported, @target_loc.scheme
+      end
+
+    rescue RestFtpDaemon::AssertionFailed => exception
+      return oops :started, exception
+
+    # rescue URI::InvalidURIError => exception
+    #   return oops :started, exception, "target_invalid"
+    end
+
+    def work
+      # Scan local source files from disk
+      set_status JOB_STATUS_CHECKING_SRC
+      sources = scan_local_paths @source_loc.path
+      set_info :source, :count, sources.count
+      set_info :source, :files, sources.collect(&:full)
+      log_info "JobTransfer.work sources #{sources.collect(&:name)}"
+      raise RestFtpDaemon::SourceNotFound if sources.empty?
+
+      # Guess target file name, and fail if present while we matched multiple sources
+      raise RestFtpDaemon::TargetDirectoryError if @target_loc.name && sources.count>1
+
+      # Connect to remote server and login
+      set_status JOB_STATUS_CONNECTING
+      @remote.connect
+
+      # Prepare target path or build it if asked
+      set_status JOB_STATUS_CHDIR
+      @remote.chdir_or_create @target_loc.dir, @mkdir
+
+      # Compute total files size
+      @transfer_total = sources.collect(&:size).sum
+      set_info :work, :total, @transfer_total
+
+      # Reset counters
+      @last_data = 0
+      @last_time = Time.now
+
+      # Handle each source file matched, and start a transfer
+      source_processed = 0
+      targets = []
+      sources.each do |source|
+        # Compute target filename
+        target_final = @target_loc.clone
+
+        # Add the source file name if none found in the target path
+        unless full_target.name
+          target_final.name = source.name
+        end
+
+        # Do the transfer, for each file
+        remote_push source, target_final
+
+        # Add it to transferred target names
+        targets << target_final.path
+        set_info :target, :files, targets
+
+        # Update counters
+        set_info :source, :processed, source_processed += 1
+      end
+
+    rescue SocketError => exception
+      return oops :ended, exception, "conn_socket_error"
+
+    rescue EOFError => exception
+      return oops :ended, exception, "conn_eof"
+
+    rescue Errno::EHOSTDOWN => exception
+      return oops :ended, exception, "conn_host_is_down"
+
+    rescue Errno::EPIPE=> exception
+      return oops :ended, exception, "conn_broken_pipe"
+
+    rescue Errno::ENETUNREACH => exception
+      return oops :ended, exception, "conn_unreachable"
+
+    rescue Errno::ECONNRESET => exception
+      return oops :ended, exception, "conn_reset_by_peer"
+
+    rescue Errno::ENOTCONN => exception
+      return oops :ended, exception, "conn_failed"
+
+    rescue Errno::ECONNREFUSED => exception
+      return oops :ended, exception, "conn_refused"
+
+    rescue Timeout::Error, Errno::ETIMEDOUT, Net::ReadTimeout => exception
+      return oops :ended, exception, "conn_timed_out"
+
+    rescue OpenSSL::SSL::SSLError => exception
+      return oops :ended, exception, "conn_openssl_error"
+
+    rescue Net::FTPReplyError => exception
+      return oops :ended, exception, "ftp_reply_error"
+
+    rescue Net::FTPTempError => exception
+      return oops :ended, exception, "ftp_temp_error"
+
+    rescue Net::FTPPermError => exception
+      return oops :ended, exception, "ftp_perm_error"
+
+    rescue Net::FTPProtoError => exception
+      return oops :ended, exception, "ftp_proto_error"
+
+    rescue Net::FTPError => exception
+      return oops :ended, exception, "ftp_error"
+
+    rescue Net::SFTP::StatusException => exception
+      return oops :ended, exception, "sftp_exception"
+
+    rescue Net::SSH::HostKeyMismatch => exception
+      return oops :ended, exception, "sftp_key_mismatch"
+
+    rescue Net::SSH::AuthenticationFailed => exception
+      return oops :ended, exception, "sftp_auth_failed"
+
+    rescue Errno::EMFILE => exception
+      return oops :ended, exception, "too_many_open_files"
+
+    rescue Errno::EINVAL => exception
+      return oops :ended, exception, "invalid_argument", true
+
+    # rescue Encoding::UndefinedConversionError => exception
+    #   return oops :ended, exception, "encoding_error", true
+
+    rescue RestFtpDaemon::SourceNotFound => exception
+      return oops :ended, exception
+
+    rescue RestFtpDaemon::TargetFileExists => exception
+      return oops :ended, exception
+
+    rescue RestFtpDaemon::TargetDirectoryError => exception
+      return oops :ended, exception
+
+    rescue RestFtpDaemon::TargetPermissionError => exception
+      return oops :ended, exception
+
+    rescue RestFtpDaemon::AssertionFailed => exception
+      return oops :ended, exception
+    end
+
+    def after
+      # Close FTP connexion and free up memory
+      log_info "JobTransfer.after"
+      @remote.close
+
+      # Free-up remote object
+      @remote = nil
+
+      # Update job status
+      set_status JOB_STATUS_DISCONNECTING
+      @finished_at = Time.now
+
+      # Update counters
+      RestFtpDaemon::Counters.instance.increment :jobs, :finished
+      RestFtpDaemon::Counters.instance.add :data, :transferred, @transfer_total
+    end
+
+    def remote_push source, target
+      # Method assertions
+      raise RestFtpDaemon::AssertionFailed, "remote_push/remote" if @remote.nil?
+      raise RestFtpDaemon::AssertionFailed, "remote_push/source" if source.nil?
+      raise RestFtpDaemon::AssertionFailed, "remote_push/target" if target.nil?
+
+      # Use source filename if target path provided none (typically with multiple sources)
+      log_info "JobTransfer.remote_push [#{source.name}]: [#{source.full}] > [#{target.full}]"
+      set_info :source, :current, source.name
+
+      # Compute temp target name
+      tempname = nil
+      if @tempfile
+        tempname = "#{target.name}.temp-#{identifier(JOB_TEMPFILE_LEN)}"
+        log_debug "JobTransfer.remote_push tempname [#{tempname}]"
+      end
+
+      # Remove any existing version if expected, or test its presence
+      if @overwrite
+        @remote.remove! target
+      elsif size = @remote.present?(target)
+        log_debug "JobTransfer.remote_push existing (#{format_bytes size, 'B'})"
+        raise RestFtpDaemon::TargetFileExists
+      end
+
+      # Start transfer
+      transfer_started_at = Time.now
+      @progress_at = 0
+      @notified_at = transfer_started_at
+
+      # Start the transfer, update job status after each block transfer
+      set_status JOB_STATUS_UPLOADING
+      @remote.push source, target, tempname do |transferred, name|
+        # Update transfer statistics
+        progress transferred, name
+
+        # Touch my worker status
+        touch_job
+      end
+
+      # Compute final bitrate
+      global_transfer_bitrate = get_bitrate @transfer_total, (Time.now - transfer_started_at)
+      set_info :work, :bitrate, global_transfer_bitrate.round(0)
+
+      # Done
+      set_info :source, :current, nil
+    end
+
+    def progress transferred, name = ""
+
+
+      # What's current time ?
+      now = Time.now
+      notify_after = @config[:notify_after]
+
+      # Update counters
+      @transfer_sent += transferred
+      set_info :work, :sent, @transfer_sent
+
+      # Update job info
+      percent0 = (100.0 * @transfer_sent / @transfer_total).round(0)
+      set_info :work, :progress, percent0
+
+      # Update job status after each NOTIFY_UPADE_STATUS
+      progressed_ago = (now.to_f - @progress_at.to_f)
+      if (!JOB_UPDATE_INTERVAL.to_f.zero?) && (progressed_ago > JOB_UPDATE_INTERVAL.to_f)
+        @current_bitrate = running_bitrate @transfer_sent
+        set_info :work, :bitrate, @current_bitrate.round(0)
+
+        # Log progress
+        stack = []
+        stack << "#{percent0} %"
+        stack << (format_bytes @transfer_sent, "B")
+        stack << (format_bytes @transfer_total, "B")
+        stack << (format_bytes @current_bitrate.round(0), "bps")
+        stack2 = stack.map { |txt| ("%#{LOG_PIPE_LEN.to_i}s" % txt) }.join("\t")
+        log_debug "progress #{stack2} \t#{name}"
+
+        # Remember when we last did it
+        @progress_at = now
+      end
+
+      # Notify if requested
+      notified_ago = (now.to_f - @notified_at.to_f)
+      if (!notify_after.nil?) && (notified_ago > notify_after)
+        # Prepare and send notification
+        notif_status = {
+          progress: percent0,
+          transfer_sent: @transfer_sent,
+          transfer_total: @transfer_total,
+          transfer_bitrate: @current_bitrate.round(0),
+          }
+        client_notify :progress, status: notif_status
+
+        # Remember when we last did it
+        @notified_at = now
+      end
+    end
+
+  private
+
+    def get_bitrate delta_data, delta_time
+      return nil if delta_time.nil? || delta_time.zero?
+      8 * delta_data.to_f.to_f / delta_time
+    end
+
+    def running_bitrate current_data
+      return if @last_time.nil?
+
+      # Compute deltas
+      @last_data ||= 0
+      delta_data = current_data - @last_data
+      delta_time = Time.now - @last_time
+
+      # Update counters
+      @last_time = Time.now
+      @last_data = current_data
+
+      # Return bitrate
+      get_bitrate delta_data, delta_time
+    end
+  end
+
+  # NewRelic instrumentation
+  # add_transaction_tracer :prepare,        category: :task
+  # add_transaction_tracer :run,            category: :task
+
+end

data/lib/rest-ftp-daemon/jobs/video.rb

@@ -0,0 +1,107 @@
+# FIXME: handle overwrite
+# FIXME: progress over multiple files
+# FIXME: open movie files to guess total runtime
+# FIXME: analyze media files at prepare
+
+require 'streamio-ffmpeg'
+
+module RestFtpDaemon
+  class JobVideo < Job
+
+    # Process job
+    def before
+      log_info "JobVideo.before source_loc.path: #{@source_loc.path}"
+      log_info "JobVideo.before target_loc.path: #{@target_loc.path}"
+
+      # Ensure source and target are FILE
+      raise RestFtpDaemon::SourceNotSupported, @source_loc.scheme   unless source_uri.is_a? URI::FILE
+      raise RestFtpDaemon::TargetNotSupported, @target.scheme       unless target_uri.is_a? URI::FILE
+    end
+
+    def work
+      # Guess source files from disk
+      set_status JOB_STATUS_TRANSFORMING
+      sources = scan_local_paths @source_loc.path
+      raise RestFtpDaemon::SourceNotFound if sources.empty?
+
+      # Add the source file name if none found in the target path
+      target_final = @target_loc.clone
+      target_final.name = @source_loc.name unless target_final.name
+      log_info "JobVideo.work target_final.path [#{target_final.path}]"
+
+      # Ensure target directory exists
+      log_info "JobVideo.work mkdir_p [#{@target_loc.dir}]"
+      FileUtils.mkdir_p @target_loc.dir
+
+      # Do the work, for each file
+      set_info :source, :current, @source_loc.name
+      video_command @source_loc, target_final
+
+      # Done
+      set_info :source, :current, nil
+
+    rescue FFMPEG::Error => exception
+      return oops :ended, exception, "ffmpeg_error"
+    end
+
+    def after
+      # Done
+      set_info :source, :current, nil
+    end
+
+  protected
+
+    def video_command source, target
+      log_info "JobVideo.video_command [#{source.name}]: [#{source.path}] > [#{target.path}]"
+      set_info :source, :current, source.name
+
+      # Read info about source file
+      movie = FFMPEG::Movie.new(source.path)
+
+      # Build options
+      ffmpeg_custom_options = {
+        audio_codec: @video_ac,
+        video_codec: @video_vc,
+        custom: ffmpeg_custom_option_array,
+        }
+      set_info :work, :ffmpeg_custom_options, ffmpeg_custom_options
+
+      # Build command
+      movie.transcode(target.path, ffmpeg_custom_options) do |ffmpeg_progress|
+        set_info :work, :ffmpeg_progress, ffmpeg_progress
+
+        percent0 = (100.0 * ffmpeg_progress).round(0)
+        set_info :work, :progress, percent0
+
+        log_debug "progress #{ffmpeg_progress}"
+      end
+    end
+
+    def ffmpeg_custom_option_array
+      # Ensure options ar in the correct format
+      return [] unless @video_custom.is_a? Hash
+      # video_custom_parts = @video_custom.to_s.scan(/(?:\w|"[^"]*")+/)
+
+      # Build the final array
+      custom_parts = []
+      @video_custom.each do |name, value|
+        custom_parts << "-#{name}"
+        custom_parts << value.to_s
+      end
+
+      # Return this
+      return custom_parts
+    end
+
+  end
+end
+
+
+# require "stringio"
+# def capture_stderr
+#   real_stderr, $stderr = $stderr, StringIO.new
+#   yield
+#   $stderr.string
+# ensure
+#   $stderr = real_stderr
+# end