rest-ftp-daemon 0.422.0 → 0.423.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b007f413aeb958a7261cf4b52ab1cc55e5090295
-  data.tar.gz: a632d6c795c89047868b080308fdefc458887ed3
+  metadata.gz: ee5c41955196c0b319075e59701f5a4632239c7d
+  data.tar.gz: 7821497a1f9810b6c555e4f60c3c5e5ff5fa87c4
 SHA512:
-  metadata.gz: 004961d712ba67aff69de7732f5d9bfb04ca66783860185670784498e400502704440e44b4c1368d4f652328ba3ad948f7ec8b37543536d4211c3674f5bd5e78
-  data.tar.gz: 949d07757d921d4d6da41f5f67e7418de536d0e44c04673629b087585724105659e94709ab20178cf34166c017b83a7e265e7aee15391d18e910962ed37497b8
+  metadata.gz: 456fe58b0004eae8d1cca760dd19f9455ef6012c9352bcabcdb4e9431862c8e6a7ac673b5219b5769bbe780a05a6400f9aa694eb51813d8bbdc6b99855fbf072
+  data.tar.gz: 5aa70020d1461e97c256071281e616c6f10c297428d4111203fe624fb83c12ccc6d08c24109f3e311b3dd2035b44154601972788c4329573e792f3a1d687afd4

data/Gemfile

@@ -1,4 +1,4 @@
 source "http://rubygems.org"
 
 gemspec
-# gem 'bmc-daemon-lib', :path => '../bmc-daemon-lib'
+gem 'bmc-daemon-lib', :path => '../bmc-daemon-lib'

data/Gemfile.lock

@@ -1,7 +1,13 @@
+PATH
+  remote: ../bmc-daemon-lib
+  specs:
+    bmc-daemon-lib (0.3.4)
+      chamber (~> 2.9)
+
 PATH
   remote: .
   specs:
-    rest-ftp-daemon (0.422.0)
+    rest-ftp-daemon (0.423.0)
       activesupport (~> 4.2)
       api-auth
       aws-sdk-resources (~> 2)
@@ -47,8 +53,6 @@ GEM
       descendants_tracker (~> 0.0.4)
       ice_nine (~> 0.11.0)
       thread_safe (~> 0.3, >= 0.3.1)
-    bmc-daemon-lib (0.3.4)
-      chamber (~> 2.9)
     builder (3.2.2)
     chamber (2.9.1)
       hashie (~> 3.3)
@@ -144,7 +148,7 @@ GEM
       rspec-core (~> 3.5.0)
       rspec-expectations (~> 3.5.0)
       rspec-mocks (~> 3.5.0)
-    rspec-core (3.5.2)
+    rspec-core (3.5.3)
       rspec-support (~> 3.5.0)
     rspec-expectations (3.5.0)
       diff-lcs (>= 1.2.0, < 2.0)
@@ -189,6 +193,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  bmc-daemon-lib!
   bundler (~> 1.6)
   http
   pry

data/README.md

@@ -353,7 +353,7 @@ Known bugs
 
 * As this project is based on Chamber, and it considers hyphens in filename as namespaces, the global /etc/rest-ftp-daemon.yml config file is not parsed (and thus, ignored). Until this is worked around, please specify a config filename on the commandline.
 
-* If you get ```fatal error: 'openssl/ssl.h' file not found when installing ```eventmachine``` on OSX El Capitan, you can try with:
+* If you get ```fatal error: 'openssl/ssl.h' file not found when installing eventmachine``` on OSX El Capitan, you can try with:
 ```
 gem install eventmachine -v '1.0.8' -- --with-cppflags=-I/usr/local/opt/openssl/include
 bundle install

data/lib/rest-ftp-daemon/api/debug.rb

@@ -24,7 +24,7 @@ module RestFtpDaemon
             me[:error] = job.error.encoding.to_s unless job.error.nil?
             me[:status] = job.status.encoding.to_s unless job.status.nil?
 
-            RestFtpDaemon::Job::FIELDS.each do |name|
+            RestFtpDaemon::Job::IMPORTED.each do |name|
               value = job.send(name)
               me[name] = value.encoding.to_s if value.is_a? String
             end

data/lib/rest-ftp-daemon/api/entities/job.rb

@@ -1,59 +1,57 @@
 require "grape-entity"
 
 module RestFtpDaemon
-  module API
-    module Entities
-      class Job < Grape::Entity
+  module Entities
+    class Job < Grape::Entity
 
-        # Job ID
-        expose :id
-
-        # Job specific attributes and flags
-        RestFtpDaemon::Job::FIELDS.each { |name| expose name }
-
-        # Technical fields
-        expose :wid, unless: lambda { |object, _options| object.wid.nil? }
-
-        # expose :error
-        expose :json_error, as: :error
-        expose :json_status, as: :status
-        #expose :json_target, as: :target_method
+      # Some formatters
+      format_with(:utf8_filter) do |thing|
+        thing.to_s.encode("UTF-8") if thing
+      end
 
-        expose :queued_at
-        expose :updated_at
-        expose :started_at
-        expose :finished_at
+      # Job-execution related
+      expose :id
+      expose :wid, unless: lambda { |object, _options| object.wid.nil? }
 
-        # Computed fields
-        expose :age
-        expose :exectime
+      # Attributes from API
+      RestFtpDaemon::Job::IMPORTED.each do |field|
+        expose field
+      end
 
-        # Params
-        expose :infos, unless: :hide_infos
+      # Work-specific options
+      expose :overwrite
+      expose :mkdir
+      expose :tempfile
+      expose :video_custom
 
-        # Options
-        # expose :options, using: API::Entities::Options
-        # expose :video_ac
-        # expose :video_custom
+      # Job/Video options
+      expose :video_options
+      #, using: Entities::VideoOptions
 
-        # with_options(format_with: :iso_timestamp) do
-        #     expose :created_at
-        #     expose :updated_at
-        # end
+      # Status and error
+      expose :status, format_with: :utf8_filter
+      expose :error, format_with: :utf8_filter
 
-        # expose :age do
-        # end
+      # Time stamps
+      expose :queued_at
+      expose :updated_at
+      expose :started_at
+      expose :finished_at
 
-        # expose :slots do |station,options|
-        #   station.slots.map{ |slot| SlotEntity.new(slot).serializable_hash }
-        # end
+      # Computed fields
+      expose :age           #, safe: true
+      expose :exectime
 
-      private
+      # Infos
+      expose :infos, unless: :hide_infos
 
-        def priv_func
-        end
+      # Source and target     #, :unless => Proc.new {|g| g.source_loc.nil?}
+      expose :source_loc, using: Entities::Location#, as: :source
+      expose :target_loc, using: Entities::Location#, as: :target
 
-      end
+      # expose :slots do |station,options|
+      #   station.slots.map{ |slot| SlotEntity.new(slot).serializable_hash }
+      # end
     end
   end
 end

data/lib/rest-ftp-daemon/api/entities/location.rb

@@ -0,0 +1,25 @@
+require "grape-entity"
+
+module RestFtpDaemon
+  module Entities
+    class Location < Grape::Entity
+
+      expose :original, as: '_'
+      expose :uri
+      expose :scheme
+
+      expose :host, unless: Proc.new {|obj| obj.host.nil?}
+      expose :user, unless: Proc.new {|obj| obj.user.nil?}
+      expose :port, if: Proc.new {|obj| !obj.port.nil?}
+
+      expose :dir
+      expose :name
+      expose :path
+
+      expose :aws_region ,unless: Proc.new {|obj| obj.aws_region.nil?}
+      expose :aws_bucket, unless: Proc.new {|obj| obj.aws_bucket.nil?}
+      expose :aws_id,     unless: Proc.new {|obj| obj.aws_id.nil?}
+
+    end
+  end
+end

data/lib/rest-ftp-daemon/api/entities/options.rb

@@ -1,15 +1,13 @@
 require "grape-entity"
 
 module RestFtpDaemon
-  module API
-    module Entities
-      class Options < Grape::Entity
+  module Entities
+    class Options < Grape::Entity
 
-        # expose :opt1, documentation: { type: 'Boolean', desc: 'opt UN', required: false }
-        expose :opt2
-        # expose :opt6
+      # expose :opt1, documentation: { type: 'Boolean', desc: 'opt UN', required: false }
+      expose :opt2
+      # expose :opt6
 
-      end
     end
   end
 end

data/lib/rest-ftp-daemon/api/jobs.rb

@@ -33,7 +33,7 @@ module RestFtpDaemon
 
         else
           status 200
-          present job, with: RestFtpDaemon::API::Entities::Job, type: "complete"
+          present job, with: RestFtpDaemon::Entities::Job, type: "complete"
 
         end
       end
@@ -53,7 +53,7 @@ module RestFtpDaemon
 
         else
           status 200
-          present jobs, with: RestFtpDaemon::API::Entities::Job
+          present jobs, with: RestFtpDaemon::Entities::Job
 
         end
       end
@@ -63,30 +63,45 @@ module RestFtpDaemon
         requires :source, type: String, desc: "Source file pattern"
         requires :target, type: String, desc: "Target remote path"
 
-        optional :label, type: String, desc: "Descriptive label for this job"
+        optional :label, type: String, desc: "Descriptive label (info only)"
         optional :notify, type: String, desc: "URL to get POST'ed notifications back"
-        optional :priority, type: Integer, desc: "Priority level of the job (lower is stronger)"
-        optional :pool, type: String, desc: "Pool of worker to be used"
         optional :type,
           type: String,
           desc: "Type of job",
           default: JOB_TYPE_TRANSFER,
           values: {value: JOB_TYPES, message: "should be one of: #{JOB_TYPES.join', '}"},
           allow_blank: { value: false, message: 'cannot be empty' }
-
-        optional :video_vc,
-          type: String,
-          desc: "video: video codec",
-          default: nil
-        optional :video_ac,
+        optional :pool,
           type: String,
-          desc: "video: audio codec",
-          default: nil
+          desc: "Pool of worker to be used",
+          default: DEFAULT_POOL
+        optional :priority,
+          type: Integer,
+          desc: "Priority level of the job (lower is stronger)",
+          default: 0
+
+        # optional :video_options,
+        #   type: Hash,
+          # type: Entities::VideoOptions,
+        #   desc: "Video: standard options passed to FFMPEG encoder",
+        #   default: {}
+
+        optional :video_options, type: Hash, desc: "Options passed to FFMPEG encoder" do
+           optional :video_codec,             type: String
+           optional :video_bitrate,           type: String
+           optional :video_bitrate_tolerance, type: String
+           optional :frame_rate,              type: Integer
+           optional :resolution,              type: String
+           optional :aspect,                  type: String
+           optional :keyframe_interval,       type: String
+           optional :x264_vprofile,           type: String
+           optional :x264_preset,             type: String
+           optional :audio_codec,             type: String
+           optional :audio_bitrate,           type: String
+           optional :audio_sample_rate,       type: Integer
+           optional :audio_channels,          type: String
+        end
 
-        optional :video_options,
-          type: Hash,
-          desc: "Video: standard options passed to FFMPEG encoder",
-          default: {}
         optional :video_custom,
           type: Hash,
           desc: "video: custom options passed to FFMPEG encoder",
@@ -104,7 +119,6 @@ module RestFtpDaemon
           type: Boolean,
           desc: "Upload to a temp file before renaming it to the target filename",
           default: Conf.at(:transfer, :tempfile)
-        # optional :options, desc: "Options passed to FFMPEG (video jobs)", type: API::Entities::Options
       end
 
       post "/" do
@@ -130,7 +144,7 @@ module RestFtpDaemon
 
         else
           status 201
-          present job, with: RestFtpDaemon::API::Entities::Job, hide_params: true
+          present job, with: RestFtpDaemon::Entities::Job, hide_params: true
 
         end
       end

data/lib/rest-ftp-daemon/api/root-real.rb

@@ -55,7 +55,7 @@ desc 'API Root'
           description: "API description for #{BmcDaemonLib::Conf.app_name} #{BmcDaemonLib::Conf.app_ver}",
           }
          # models: [
-         #   RestFtpDaemon::API::Entities::Job,
+         #   RestFtpDaemon::Entities::Job,
          # ]
 
       ### GLOBAL EXCEPTION HANDLING

data/lib/rest-ftp-daemon/api/root-test.rb

@@ -56,7 +56,7 @@ module RestFtpDaemon
 
         else
           status 200
-          present jobs, with: RestFtpDaemon::API::Entities::Job
+          present jobs, with: RestFtpDaemon::Entities::Job
 
         end
       end

data/lib/rest-ftp-daemon/constants.rb

@@ -4,10 +4,8 @@ DEFAULT_SFTP_TIMEOUT    = 600   # 10mn
 DEFAULT_FTP_CHUNK       = 1024  # 1 MB
 DEFAULT_PAGE_SIZE       = 50    # 50 lines
 DEFAULT_RETRY_AFTER     = 10    # 10s
-
 TARGET_BLANK             = "_blank"
 
-
 # Internal job constants
 JOB_RANDOM_LEN          = 8
 JOB_IDENT_LEN           = 4
@@ -17,9 +15,13 @@ JOB_UPDATE_INTERVAL     = 1
 JOB_FFMPEG_THREADS      = 2
 JOB_FFMPEG_ATTRIBUTES   = [:video_codec, :video_bitrate, :video_bitrate_tolerance, :frame_rate, :resolution, :aspect, :keyframe_interval, :x264_vprofile, :x264_preset, :audio_codec, :audio_bitrate, :audio_sample_rate, :audio_channels]
 
+# Internal job infos
+INFO_PROGRESS             = :work_progress
+INFO_BITRATE              = :transfer_bitrate
+
 # Constants: logger
 LOG_ROTATION             = "daily"
-LOG_FORMAT_PROGNAME     = "%d\t%s"
+LOG_FORMAT_PROGNAME      = "%d\t%s"
 LOG_HEADER_TIME          = "%Y-%m-%d %H:%M:%S"
 LOG_HEADER_FORMAT        = "%s \t%d\t%-8s %-10s "
 LOG_MESSAGE_TRIM         = 200

data/lib/rest-ftp-daemon/job.rb

@@ -14,15 +14,14 @@ module RestFtpDaemon
     attr_reader :logger
     include BmcDaemonLib::LoggerHelper
 
-    # Class constants
-    FIELDS = [:type, :source, :target, :label, :priority, :pool, :notify,
-      :overwrite, :mkdir, :tempfile,
-      :video_options, :video_custom
-      ]
+    # Fields to be imported from params
+    IMPORTED = %w(type priority pool label priority source target overwrite mkdir tempfile video_options video_custom)
 
     # Class options
     attr_accessor :wid
-    attr_accessor :type
+
+    attr_reader :source_loc
+    attr_reader :target_loc
 
     attr_reader :id
     attr_reader :error
@@ -35,12 +34,14 @@ module RestFtpDaemon
     attr_reader :finished_at
 
     attr_reader :infos
-    attr_reader :pool
+    #attr_reader :config
 
-    attr_accessor :config
+    # delegate :type, :pool, :label, :priority,
+    #   to: :params
 
-    FIELDS.each do |name|
-      attr_reader name
+    # Define readers from imported fields
+    IMPORTED.each do |field|
+      attr_reader field
     end
 
     def initialize job_id = nil, params = {}
@@ -52,14 +53,14 @@ module RestFtpDaemon
       return if job_id.nil?
 
       # Init context
-      @id = job_id.to_s
-      #@updated_at = nil
-      @started_at = nil
-      @finished_at = nil
-      @error = nil
-      @status = nil
-      @runs = 0
-      @wid = nil
+      @id           = job_id.to_s
+      @started_at   = nil
+      @finished_at  = nil
+      @updated_at   = nil
+      @error        = nil
+      @status       = nil
+      @runs         = 0
+      @wid          = nil
 
       # Prepare configuration
       @config       = Conf[:transfer] || {}
@@ -67,27 +68,33 @@ module RestFtpDaemon
       @pools        = Conf[:pools] || {}
 
       # Logger
-      @logger = BmcDaemonLib::LoggerPool.instance.get :transfer
+      @logger   = BmcDaemonLib::LoggerPool.instance.get :transfer
 
       # Import query params
-      FIELDS.each do |name|
-        instance_variable_set "@#{name}", params[name]
+      set_info :params, params
+      IMPORTED.each do |field|
+        instance_variable_set "@#{field}", params[field]
       end
 
       # Check if pool name exists
-      if (@pools.keys.include? params[:pool])
-        @pool = params[:pool].to_s
-      else
-        @pool = DEFAULT_POOL
-      end
+      @pool = DEFAULT_POOL unless @pools.keys.include?(@pool)
 
       # Prepare sources/target
-      prepare_source
-      prepare_target
+      raise RestFtpDaemon::AttributeMissing, "source" unless params[:source]
+      @source_loc = Location.new(params[:source])
+      #set_info :location_source, params[:source]
+      log_info "Job.initialize source #{@source_loc.uri}"
+
+      raise RestFtpDaemon::AttributeMissing, "target" unless params[:target]
+      @target_loc = Location.new(params[:target])
+      #set_info :location_target, params[:target]
+      log_info "Job.initialize target #{@target_loc.uri}"
 
       # Handle exceptions
-      rescue RestFtpDaemon::UnsupportedScheme => exception
-        return oops :started, exception
+      # rescue RestFtpDaemon::UnresolvedTokens => exception
+      #   return oops :started, exception
+      # rescue RestFtpDaemon::UnsupportedScheme => exception
+      #   return oops :started, exception
     end
 
     def reset
@@ -101,8 +108,6 @@ module RestFtpDaemon
       # Job has been prepared, reset infos
       set_status JOB_STATUS_PREPARED
       @infos = {}
-      set_info_location :source, @source_loc
-      set_info_location :target, @target_loc
 
       # Update job status, send first notification
       set_status JOB_STATUS_QUEUED
@@ -156,11 +161,11 @@ module RestFtpDaemon
     end
 
     def source_uri
-      @source_loc.uri
+      @source_loc.uri if @source_loc
     end
 
     def target_uri
-      @target_loc.uri
+      @target_loc.uri if @target_loc
     end
 
     def weight
@@ -190,41 +195,20 @@ module RestFtpDaemon
     end
 
     def targethost
-      get_info :target, :host
-    end
-
-    # def json_target
-    #   utf8 get_info(:target, :method)
-    # end
-
-    def json_error
-      utf8 @error unless @error.nil?
-    end
-
-    def json_status
-      utf8 @status unless @status.nil?
+      @target_loc.host unless @target_loc.nil?
+      #get_info :target_host
     end
 
-    def get_info level1, level2
+     def get_info name
       @mutex.synchronize do
-        # @infos || {}
-        @infos[level1][level2] if @infos[level1].is_a? Hash
+        @infos[name]
       end
     end
 
-    def set_info level1, level2, value
+    def set_info name, value
       @mutex.synchronize do
         @infos || {}
-        @infos[level1] ||= {}
-
-        # Force strings to UTF8
-        if value.is_a? Symbol
-          @infos[level1][level2] = value.to_s.force_encoding(Encoding::UTF_8)
-        elsif value.is_a? String
-          @infos[level1][level2] = value.dup.force_encoding(Encoding::UTF_8)
-        else
-          @infos[level1][level2] = value
-        end
+        @infos[name] = debug_value_utf8(value)
       end
       touch_job
     end
@@ -235,28 +219,6 @@ module RestFtpDaemon
       log_error "Job PLACEHOLDER METHOD CALLED"
     end
 
-    def prepare_source
-      raise RestFtpDaemon::AttributeMissing, "source" unless @source
-      @source_loc = Location.new @source
-      log_info "Job.prepare_source #{@source_loc.uri}"
-    end
-
-    def prepare_target
-      raise RestFtpDaemon::AttributeMissing, "target" unless @target
-      @target_loc = Location.new @target
-      log_info "Job.prepare_target #{@target_loc.uri}"
-    end
-
-    def set_info_location prefix, location
-      return unless location.is_a? Location
-      fields = [:uri, :scheme, :user, :host, :port, :dir, :name, :path, :aws_region, :aws_bucket, :aws_id]
-
-      # Add each field to @infos
-      fields.each do |what|
-        set_info prefix, "loc_#{what}".to_sym, location.send(what)
-      end
-    end
-
   private
 
     def log_prefix
@@ -269,6 +231,18 @@ module RestFtpDaemon
       Thread.current.thread_variable_set :updated_at, now
     end
 
+    # Force strings to UTF8
+    def debug_value_utf8 value
+      case value
+      when Symbol
+        return value.to_s.force_encoding(Encoding::UTF_8)
+      when String
+        return value.dup.force_encoding(Encoding::UTF_8) if value.is_a? String
+      else
+        return value
+      end
+    end
+
     def set_error value
       @mutex.synchronize do
         @error = value
@@ -283,10 +257,6 @@ module RestFtpDaemon
       touch_job
     end
 
-    def utf8 value
-      value.to_s.encode("UTF-8")
-    end
-
     def flag_prepare name
       # build the flag instance var name
       variable = "@#{name}"
@@ -339,13 +309,13 @@ module RestFtpDaemon
       # Update job's internal status
       set_status JOB_STATUS_FAILED
       set_error error
-      set_info :error, :exception, exception.class.to_s
-      set_info :error, :message, exception.message
+      set_info :error_exception, exception.class.to_s
+      set_info :error_message,   exception.message
 
       # Build status stack
       notif_status = nil
       if include_backtrace
-        set_info :error, :backtrace, exception.backtrace
+        set_info :error_backtrace, exception.backtrace
         notif_status = {
           backtrace: exception.backtrace,
           }

data/lib/rest-ftp-daemon/job_queue.rb

@@ -102,10 +102,9 @@ module RestFtpDaemon
         next unless job.status == JOB_STATUS_UPLOADING
 
         # Extract current rate, next if not available
-        rate = job.get_info :transfer, :bitrate
+        rate = job.get_info INFO_BITRATE
         next if rate.nil?
 
-
         # Add its current rate
         result[group] ||= 0
         result[group] += rate

data/lib/rest-ftp-daemon/jobs/transfer.rb

@@ -13,7 +13,7 @@ module RestFtpDaemon
 
       # Some init
       @transfer_sent = 0
-      set_info :source, :processed, 0
+      set_info :source_processed, 0
 
       # Ensure source is FILE
       raise RestFtpDaemon::SourceNotSupported, @source_loc.scheme   unless @source_loc.is? URI::FILE
@@ -48,8 +48,8 @@ module RestFtpDaemon
       # Scan local source files from disk
       set_status JOB_STATUS_CHECKING_SRC
       sources = @source_loc.scan_files
-      set_info :source, :count, sources.size
-      set_info :source, :files, sources.collect(&:name)
+      set_info :source_count, sources.size
+      set_info :source_files, sources.collect(&:name)
       log_info "JobTransfer.work sources #{sources.collect(&:name)}"
       raise RestFtpDaemon::SourceNotFound if sources.empty?
 
@@ -66,7 +66,7 @@ module RestFtpDaemon
 
       # Compute total files size
       @transfer_total = sources.collect(&:size).sum
-      set_info :work, :total, @transfer_total
+      set_info :transfer_total, @transfer_total
 
       # Reset counters
       @last_data = 0
@@ -85,10 +85,10 @@ module RestFtpDaemon
 
         # Add it to transferred target names
         targets << target.name
-        set_info :target, :files, targets
+        set_info :target_files, targets
 
         # Update counters
-        set_info :source, :processed, source_processed += 1
+        set_info :source_processed, source_processed += 1
       end
     end
 
@@ -116,7 +116,7 @@ module RestFtpDaemon
 
       # Use source filename if target path provided none (typically with multiple sources)
       log_info "JobTransfer.remote_upload [#{source.name}]: [#{source.path}] > [#{target.path}]"
-      set_info :source, :current, source.name
+      set_info :source_current, source.name
 
       # Remove any existing version if present, or check if it's there
       if @overwrite
@@ -145,10 +145,10 @@ module RestFtpDaemon
 
       # Compute final bitrate
       global_transfer_bitrate = get_bitrate @transfer_total, (Time.now - transfer_started_at)
-      set_info :work, :bitrate, global_transfer_bitrate.round(0)
+      set_info :transfer_bitrate, global_transfer_bitrate.round(0)
 
       # Done
-      set_info :source, :current, nil
+      set_info :source_current, nil
     end
 
     def update_progress transferred, name = ""
@@ -158,17 +158,17 @@ module RestFtpDaemon
 
       # Update counters
       @transfer_sent += transferred
-      set_info :work, :sent, @transfer_sent
+      set_info :work_sent, @transfer_sent
 
       # Update job info
       percent0 = (100.0 * @transfer_sent / @transfer_total).round(0)
-      set_info :work, :progress, percent0
+      set_info INFO_PROGRESS, percent0
 
       # Update job status after each NOTIFY_UPADE_STATUS
       progressed_ago = (now.to_f - @progress_at.to_f)
       if (!JOB_UPDATE_INTERVAL.to_f.zero?) && (progressed_ago > JOB_UPDATE_INTERVAL.to_f)
         @current_bitrate = running_bitrate @transfer_sent
-        set_info :work, :bitrate, @current_bitrate.round(0)
+        set_info :transfer_bitrate, @current_bitrate.round(0)
 
         # Log progress
         stack = []

data/lib/rest-ftp-daemon/jobs/video.rb

@@ -42,29 +42,26 @@ module RestFtpDaemon
       FileUtils.mkdir_p @target_loc.dir
 
       # Do the work, for each file
-      set_info :source, :current, @source_loc.name
+      set_info :source_current, @source_loc.name
       ffmpeg_command @source_loc, target_final
 
       # Done
-      set_info :source, :current, nil
+      set_info :source_current, nil
     end
 
     def do_after
       # Done
-      set_info :source, :current, nil
+      set_info :source_current, nil
     end
 
     def ffmpeg_command source, target
-      set_info :source, :current, source.name
+      set_info :source_current, source.name
 
       # Read info about source file
       movie = FFMPEG::Movie.new(source.path)
-      set_info :source, :ffmpeg_size, movie.size
-      set_info :source, :ffmpeg_duration, movie.duration
-      set_info :source, :ffmpeg_resolution, movie.resolution
-
-
-
+      set_info :ffmpeg_size, movie.size
+      set_info :ffmpeg_duration, movie.duration
+      set_info :ffmpeg_resolution, movie.resolution
 
       # Build options
       options = {
@@ -74,7 +71,7 @@ module RestFtpDaemon
       JOB_FFMPEG_ATTRIBUTES.each do |name|
         options[name] = @video_options[name] unless @video_options[name].nil?
       end
-      set_info :work, :ffmpeg_options, options
+      set_info :work_ffmpeg_options, options
 
       # Announce context
       log_info "JobVideo.ffmpeg_command [#{FFMPEG.ffmpeg_binary}] [#{source.name}] > [#{target.name}]", options
@@ -82,7 +79,7 @@ module RestFtpDaemon
       # Build command
       movie.transcode(target.path, options) do |ffmpeg_progress|
         # set_info :work, :ffmpeg_progress, ffmpeg_progress
-        set_info :work, :progress, (100.0 * ffmpeg_progress).round(1)
+        set_info INFO_PROGRESS, (100.0 * ffmpeg_progress).round(1)
         log_debug "progress #{ffmpeg_progress}"
       end
     end

data/lib/rest-ftp-daemon/location.rb

@@ -8,6 +8,7 @@ module RestFtpDaemon
     # Accessors
     attr_accessor :name
 
+    attr_reader :original
     attr_reader :uri
     attr_reader :scheme
     attr_reader :dir
@@ -25,9 +26,10 @@ module RestFtpDaemon
     delegate :scheme, :host, :port, :user, :password, :to_s,
       to: :uri
 
-    def initialize path
+    def initialize original
       # Strip spaces before/after, copying original "path" at the same time
-      location_uri = path.strip
+      @original = original
+      location_uri = original.strip
 
       # Replace tokens, fix scheme for local paths
       resolve_tokens! location_uri
@@ -60,6 +62,7 @@ module RestFtpDaemon
       return @name if @dir.nil?
       File.join(@dir.to_s, @name.to_s)
     end
+    alias :to_s :path
 
     def scan_files
       Dir.glob(path).collect do |file|
@@ -114,7 +117,7 @@ module RestFtpDaemon
       @uri = URI.parse path # rescue nil
       raise RestFtpDaemon::LocationParseError, location_path unless @uri
 
-      # Sanitize path
+      # Path cleanup
       cleaned  = @uri.path.clone
 
       # remove_leading_slashes

data/lib/rest-ftp-daemon/remote.rb

@@ -6,7 +6,6 @@ module RestFtpDaemon
     # Class options
     attr_reader :logger
     attr_reader :log_prefix
-
     attr_accessor :job
 
     # Delegate set_info info to Job

data/lib/rest-ftp-daemon/remote_ftp.rb

@@ -16,7 +16,7 @@ module RestFtpDaemon
         prepare_ftp
       end
       @ftp.passive = true
-      @ftp.debug_mode =  @debug
+      @ftp.debug_mode = @debug
 
       # Config
       @chunk_size = DEFAULT_FTP_CHUNK.to_i * 1024

data/lib/rest-ftp-daemon/remote_s3.rb

@@ -4,12 +4,15 @@ require 'aws-sdk-resources'
 module RestFtpDaemon
   class RemoteS3 < Remote
 
+    MULTIPART_THRESHOLD_MB = 4
+
     # Class options
     attr_reader :client
     attr_reader :target
 
     def prepare
-      log_debug "RemoteS3.prepare target[#{@target.inspect}]"
+      @multipart_threshold = MULTIPART_THRESHOLD_MB.to_i * 1024 * 1024
+      log_debug "RemoteS3.prepare target[#{@target.inspect}] #{@multipart_threshold}"
     end
 
     def connect
@@ -18,14 +21,13 @@ module RestFtpDaemon
       # Connect init
       log_debug "RemoteS3.connect region[#{target.aws_region}] id[#{target.aws_id}]"
 
-      # Debug level
-      verbosity =  @debug ? Logger::DEBUG : false
-
       # Connect remote server
       @client = Aws::S3::Resource.new(
         region: @target.aws_region,
-        credentials: Aws::Credentials.new(@target.aws_id, @target.aws_secret)
+        credentials: Aws::Credentials.new(@target.aws_id, @target.aws_secret),
+        http_wire_trace: @debug
         )
+      #s3 = Aws::S3::Client.new(http_wire_trace: true)
     end
 
     def upload source, target, use_temp_name = false, &callback
@@ -35,12 +37,25 @@ module RestFtpDaemon
 
       # Update progress before
       #yield 0, target.name
-
-      # Do the transfer
+      # Point to the right bucket and object
       bucket = @client.bucket(target.aws_bucket)
       object = bucket.object(target.name)
-      object.upload_file source.path do |progress, total|
-        log_debug "- progress[#{progress}] total[#{total}]"
+
+      # Do the transfer
+      object.upload_file(source.path, {
+        multipart_threshold: @multipart_threshold
+        })
+
+      # Wait for transfer to complete
+      object.wait_until_exists do |waiter|
+        # waiter.delay = 1
+        # # log_debug "- progress[#{progress}] total[#{total}]"
+        # waiter.before_wait do |attempts, response|
+        #   puts "#{attempts} made"
+        #   puts response.error.inspect
+        #   puts response.data.inspect
+        # end
+        # log_debug "- progress[] #{waiter.inspect}"
       end
 
       # Update progress after
@@ -49,8 +64,8 @@ module RestFtpDaemon
       # Dump information about this file
       log_debug "RemoteS3.upload url[#{object.public_url}]"
       log_debug "RemoteS3.upload etag[#{object.etag}]"
-      set_info :target, :aws_public_url, object.public_url
-      set_info :target, :aws_etag, object.etag
+      set_info :target_aws_public_url, object.public_url
+      set_info :target_aws_etag, object.etag
     end
 
     def connected?

data/lib/rest-ftp-daemon/uri.rb

@@ -1,5 +1,11 @@
 module URI
 
+  class FILE < Generic
+  end
+
+  class S3 < Generic
+  end
+
   class FTPS < Generic
     DEFAULT_PORT = 21
   end
@@ -12,12 +18,6 @@ module URI
     DEFAULT_PORT = 22
   end
 
-  class S3 < Generic
-  end
-
-  class FILE < Generic
-    DEFAULT_PORT = 22
-  end
 
   @@schemes["FTPS"] = FTPS
   @@schemes["FTPES"] = FTPES

data/lib/rest-ftp-daemon/views/dashboard_table.haml

@@ -1,12 +1,14 @@
 -# coding: utf-8
 - jobs.each do |job|
-  - work_progress     = job.get_info(:work, :progress)
-  - work_bitrate      = job.get_info(:work, :bitrate)
-  - work_sent         = job.get_info(:work, :sent)
+  - work_progress     = job.get_info(INFO_PROGRESS)
+  - error_message     = job.get_info(:error_message)
+  - transfer_total    = job.get_info(:transfer_total)
+  - transfer_bitrate  = job.get_info(:transfer_bitrate)
+  - work_sent         = job.get_info(:work_sent)
 
-  - source_count      = job.get_info(:source, :count) || 0
-  - source_processed  = job.get_info(:source, :processed) || 0
-  - source_current    = job.get_info(:source, :current)
+  - source_count      = job.get_info(:source_count) || 0
+  - source_processed  = job.get_info(:source_processed) || 0
+  - source_current    = job.get_info(:source_current)
 
   - job_working       = [JOB_STATUS_UPLOADING, JOB_STATUS_TRANSFORMING].include? job.status
 
@@ -59,16 +61,16 @@
             = "#{work_progress} %"
 
       - else
-        .error{title: job.get_info(:error, :message)}
+        .error{title: error_message}
           = text_or_empty(job.error)
 
     %td.nobr.text-right
-      = format_bytes(job.get_info(:transfer, :total), "B")
+      = format_bytes(transfer_total, "B")
 
     %td.nobr.text-right{title: "time: #{job.exectime} s"}
-      - if work_bitrate
+      - if transfer_bitrate
         %span.push-bitrate
-          = format_bytes(work_bitrate, "bps")
+          = format_bytes(transfer_bitrate, "bps")
 
     %td
       - unless job.wid.nil?

data/lib/rest-ftp-daemon.rb

@@ -50,6 +50,7 @@ require_relative "rest-ftp-daemon/workers/reporter"
 require_relative "rest-ftp-daemon/workers/transfer"
 
 # API
+require_relative "rest-ftp-daemon/api/entities/location"
 require_relative "rest-ftp-daemon/api/entities/options"
 require_relative "rest-ftp-daemon/api/entities/job"
 require_relative "rest-ftp-daemon/api/jobs"

data/rest-ftp-daemon.gemspec

@@ -2,7 +2,7 @@
 Gem::Specification.new do |spec|
 
   # Project version
-  spec.version                    = "0.422.0"
+  spec.version                    = "0.423.0"
 
   # Project description
   spec.name                       = "rest-ftp-daemon"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rest-ftp-daemon
 version: !ruby/object:Gem::Version
-  version: 0.422.0
+  version: 0.423.0
 platform: ruby
 authors:
 - Bruno MEDICI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-02 00:00:00.000000000 Z
+date: 2016-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -413,6 +413,7 @@ files:
 - lib/rest-ftp-daemon/api/dashboard.rb
 - lib/rest-ftp-daemon/api/debug.rb
 - lib/rest-ftp-daemon/api/entities/job.rb
+- lib/rest-ftp-daemon/api/entities/location.rb
 - lib/rest-ftp-daemon/api/entities/options.rb
 - lib/rest-ftp-daemon/api/jobs.rb
 - lib/rest-ftp-daemon/api/root-real.rb