rest-ftp-daemon 0.435.2 → 0.501.0

data/lib/rest-ftp-daemon/remote/s3.rb

@@ -4,76 +4,135 @@ require 'aws-sdk-resources'
 module RestFtpDaemon
   module Remote
     class RemoteS3 < RemoteBase
-
-      MULTIPART_THRESHOLD_MB = 4
+      include CommonHelpers
 
       # Class options
       attr_reader :client
       attr_reader :target
 
-      def prepare
-        @multipart_threshold = MULTIPART_THRESHOLD_MB.to_i * 1024 * 1024
-        log_debug "RemoteS3.prepare target[#{@target.inspect}] #{@multipart_threshold}"
-      end
+      # def prepare
+      # end
 
       def connect
         super
 
         # Connect init
-        log_debug "RemoteS3.connect region[#{target.aws_region}] id[#{target.aws_id}]"
+        log_debug "connect region[#{target.aws_region}] id[#{target.aws_id}]"
 
         # Connect remote server
-        @client = Aws::S3::Resource.new(
+        @client = Aws::S3::Client.new(
           region: @target.aws_region,
           credentials: Aws::Credentials.new(@target.aws_id, @target.aws_secret),
-          # thread_count: 4,
           http_wire_trace: @debug
           )
-        #s3 = Aws::S3::Client.new(http_wire_trace: true)
+      end
+
+      def size_if_exists target
+        log_debug "size_if_exists [#{target.path}]"
+        object = @client.get_object(bucket: target.aws_bucket, key: target.path)
+      rescue Aws::S3::Errors::NotFound => e
+        return false
+      else
+        return object.content_length
       end
 
       def upload source, target, use_temp_name = false, &callback
         # Push init
         raise RestFtpDaemon::AssertionFailed, "upload/client" if @client.nil?
-        log_debug "RemoteS3.upload bucket[#{target.aws_bucket}] name[#{target.name}]"
-
-        # Update progress before
-        #yield 0, target.name
-        # Point to the right bucket and object
-        bucket = @client.bucket(target.aws_bucket)
-        object = bucket.object(target.name)
-
-        # Do the transfer
-        object.upload_file(source.path, {
-          multipart_threshold: @multipart_threshold
-          })
+        log_debug "upload bucket[#{target.aws_bucket}] path[#{target.path}]"
 
-        # Wait for transfer to complete
-        object.wait_until_exists do |waiter|
-          # waiter.delay = 1
-          # # log_debug "- progress[#{progress}] total[#{total}]"
-          # waiter.before_wait do |attempts, response|
-          #   puts "#{attempts} made"
-          #   puts response.error.inspect
-          #   puts response.data.inspect
-          # end
-          # log_debug "- progress[] #{waiter.inspect}"
+        # Do the transfer, passing the file to the best method
+        File.open(source.filepath, 'r', encoding: 'BINARY') do |file|
+          if file.size >= JOB_S3_MIN_PART
+            upload_multipart  file, target.aws_bucket, target.path, target.name, &callback
+          else
+            upload_onefile    file, target.aws_bucket, target.path, target.name, &callback
+          end
         end
 
-        # Update progress after
-        #yield target.size, target.name
-
-        # Dump information about this file
-        log_debug "RemoteS3.upload url[#{object.public_url}]"
-        log_debug "RemoteS3.upload etag[#{object.etag}]"
-        set_info :target_aws_public_url, object.public_url
-        set_info :target_aws_etag, object.etag
+        # We're all set
+        log_debug "RemoteS3.upload done"
       end
 
       def connected?
         !@client.nil?
       end
 
+    private
+
+      def upload_onefile file, s3_bucket, s3_path, s3_name, &callback
+        log_debug "upload_onefile"
+        @client.put_object(bucket: s3_bucket, key: s3_path, body: file)
+      end
+
+      def upload_multipart file, s3_bucket, s3_path, s3_name, &callback
+        # Init
+        current_part  = 1
+
+        # Compute parameters
+        file_size     = file.size
+        parts_size    = compute_parts_size(file_size)
+        parts_count   = (file_size.to_f / parts_size).ceil
+        log_debug "upload_multipart", {
+          file_size:    format_bytes(file_size, "B"),
+          parts_size:   format_bytes(parts_size, "B"),
+          parts_count:  parts_count
+          }
+
+        # Prepare basic opts
+        options = {
+          bucket: s3_bucket,
+          key:    s3_path,
+          }
+
+        # Declare multipart upload
+        mpu_create_response = @client.create_multipart_upload(options)
+        options[:upload_id] = mpu_create_response.upload_id
+        log_debug "created multipart id[#{options[:upload_id]}]"
+
+        # Upload each part
+        file.each_part(parts_size) do |part|
+          # Prepare part upload
+          opts = options.merge({
+            body:        part,
+            part_number: current_part,
+            })
+          log_debug "upload_part [#{current_part}/#{parts_count}]"
+          resp = @client.upload_part(opts)  
+          
+          # Send progress info upwards
+          yield parts_size, s3_name
+
+          # Increment part number
+          current_part += 1
+        end
+
+        # Retrieve parts and complete upload
+        log_debug "complete_multipart_upload"
+        parts_resp = @client.list_parts(options)
+
+        those_parts = parts_resp.parts.map do |part| 
+          { part_number: part.part_number, etag: part.etag }
+        end
+        opts = options.merge({
+          multipart_upload: {
+            parts: those_parts
+            } 
+          })
+        mpu_complete_response = @client.complete_multipart_upload(opts)
+      end  
+
+      def compute_parts_size filesize
+        # Initial part size is minimal
+        partsize_mini = JOB_S3_MIN_PART
+
+        # Other partsize if too many blocks
+        partsize_bigf = (filesize.to_f / JOB_S3_MAX_COUNT).ceil
+
+        # Decide
+        return [partsize_mini, partsize_bigf].max
+      end
+
     end
   end
-end
+end

data/lib/rest-ftp-daemon/remote/sftp.rb

@@ -8,14 +8,14 @@ module RestFtpDaemon
       # Class options
       attr_reader :sftp
 
-      def prepare
-      end
+      # def prepare
+      # end
 
       def connect
         super
 
         # Connect init
-        log_debug "RemoteSFTP.connect [#{@target.user}]@[#{@target.host}]:[#{@target.port}]"
+        log_debug "connect [#{@target.user}]@[#{@target.host}]:[#{@target.port}]"
 
         # Debug level
         verbosity =  @debug ? Logger::DEBUG : false
@@ -30,9 +30,9 @@ module RestFtpDaemon
             )
       end
 
-      def present? target
-        log_debug "RemoteSFTP.present? [#{target.name}]"
-        stat = @sftp.stat! target.path
+      def size_if_exists target
+        log_debug "size_if_exists [#{target.name}]"
+        stat = @sftp.stat! target.filepath
 
       rescue Net::SFTP::StatusException
         return false
@@ -41,8 +41,8 @@ module RestFtpDaemon
       end
 
       def remove! target
-        log_debug "RemoteSFTP.remove! [#{target.name}]"
-        @sftp.remove target.path
+        log_debug "remove! [#{target.name}]"
+        @sftp.remove target.filepath
 
       rescue Net::SFTP::StatusException
         log_debug "#{LOG_INDENT}[#{target.name}] file not found"
@@ -51,7 +51,7 @@ module RestFtpDaemon
       end
 
       def mkdir directory
-        log_debug "RemoteSFTP.mkdir [#{directory}]"
+        log_debug "mkdir [#{directory}]"
         @sftp.mkdir! directory
 
         rescue StandardError => ex
@@ -60,7 +60,7 @@ module RestFtpDaemon
 
       def chdir_or_create directory, mkdir = false
         # Init, extract my parent name and my own name
-        log_debug "RemoteSFTP.chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
+        log_debug "chdir_or_create mkdir[#{mkdir}] dir[#{directory}]"
         parent, _current = split_path(directory)
 
         # Access this directory
@@ -98,8 +98,8 @@ module RestFtpDaemon
         end
 
         # Do the transfer
-        log_debug "RemoteSFTP.upload temp[#{use_temp_name}] name[#{dest.name}]"
-        @sftp.upload! source.path, dest.path do |event, _uploader, *args|
+        log_debug "upload temp[#{use_temp_name}] name[#{dest.name}]"
+        @sftp.upload! source.filepath, dest.filepath do |event, _uploader, *args|
           case event
           when :open then
             # args[0] : file metadata
@@ -124,8 +124,8 @@ module RestFtpDaemon
         # Move the file back to its original name
         if use_temp_name
           flags = 0x00000001
-          log_debug "RemoteSFTP.upload rename [#{dest.name}] > [#{target.name}]"
-          @sftp.rename! dest.path, target.path, flags
+          log_debug "upload rename [#{dest.name}] > [#{target.name}]"
+          @sftp.rename! dest.filepath, target.filepath, flags
         end
 
         # progress:
@@ -143,4 +143,4 @@ module RestFtpDaemon
 
     end
   end
-end
+end

data/lib/rest-ftp-daemon/static/css/main.css

@@ -149,16 +149,46 @@ body {
 
 
 
-
-
-
 .indicators .btn {
   cursor: pointer;
   opacity: 1;
   filter: alpha(opacity=100);
+  }
+
+.label-group .label {
+  /*border: 1px solid red;*/
+}
+
+.label-group>.label:not(:last-child) {
+  border-bottom-right-radius: 0; 
+  border-top-right-radius: 0;
+  margin-right: 0;
+  }
+
+.label-group>.label:not(:first-child) {
+  border-bottom-left-radius: 0; 
+  border-top-left-radius: 0; 
+  margin-left: 0;
+  }
+
+.label.label-simple {
+  color: black;
+  border: 1px solid silver;
+  background-color: white;
+  }
+/*
+.label-group>.label:first-child:not(:last-child) {
+     border-bottom-right-radius: 0; 
+     border-top-right-radius: 0; 
+}
+
+.label-group>.label:last-child:not(:first-child) {
+     border-bottom-left-radius: 0; 
+     border-top-left-radius: 0; 
 }
+*/
 
 .transfer-type {
-  width: 40px;
+  /*width: 40px;*/
   }
 

data/lib/rest-ftp-daemon/uri.rb

@@ -23,4 +23,4 @@ module URI
   @@schemes["SFTP"]   = SFTP
   @@schemes["S3"]     = S3
   @@schemes["FILE"]   = FILE
-end
+end

data/lib/rest-ftp-daemon/views/dashboard.haml

@@ -40,4 +40,4 @@
 
     .footer
       .container-fluid
-        = render :dashboard_footer
+        = render :dashboard_footer

data/lib/rest-ftp-daemon/views/dashboard_counters.haml

@@ -22,4 +22,4 @@
         %tr
           %td
           %td= group
-          %td.text-right= values
+          %td.text-right= values

data/lib/rest-ftp-daemon/views/dashboard_footer.haml

@@ -39,4 +39,4 @@
 
   .btn-group.btn-group-sm
     .btn.btn-default.btn-info.disabled  Started
-    .btn.btn-default.disabled= datetime_short(Conf.app_started)
+    .btn.btn-default.disabled= datetime_short(Conf.app_started)

data/lib/rest-ftp-daemon/views/dashboard_header.haml

@@ -35,4 +35,4 @@
 
   .btn-group.btn-group-sm
     .btn.btn-default.btn-success.disabled  Transferred
-    .btn.btn-default.disabled= format_bytes(info_trans, "B", 1)
+    .btn.btn-default.disabled= format_bytes(info_trans, "B", 1)

data/lib/rest-ftp-daemon/views/dashboard_jobs.haml

@@ -56,5 +56,4 @@
 
         - unless jobs.empty?
           %tbody.jobs
-            = render :dashboard_table, {jobs: jobs}
-
+            = render :dashboard_table, {jobs: jobs}

data/lib/rest-ftp-daemon/views/dashboard_rates.haml

@@ -25,4 +25,4 @@
           %td
             = group
           %td.text-right
-            = format_bytes(rate, "bps")
+            = format_bytes(rate, "bps")

data/lib/rest-ftp-daemon/views/dashboard_table.haml

@@ -34,12 +34,15 @@
     %td= job.label
 
     %td
-      = location_label job.source_uri
-      = token_highlight job.source
+      = location_label job.source_loc
+      = job.source_uri.path
+      =# token_highlight job.source
 
     %td
-      = location_label job.target_uri
-      = token_highlight job.target
+      = location_label job.target_loc
+      = job.target_uri.path
+
+      =# token_highlight job.target
 
     %td
       %span.push-status
@@ -85,4 +88,4 @@
         .label.label-default.flag.worker-label= job.priority
 
     %td
-      .label.flag.worker-label{class: job_tentatives_style(job.tentatives)}= job.tentatives
+      .label.flag.worker-label{class: job_tentatives_style(job.tentatives)}= job.tentatives

data/lib/rest-ftp-daemon/views/dashboard_tokens.haml

@@ -3,4 +3,4 @@
 
 %h2 Endpoint tokens
 - tokens.each do |name, value|
-  = token_to_label name, value
+  = token_to_label name, value

data/lib/rest-ftp-daemon/views/dashboard_workers.haml

@@ -34,4 +34,4 @@
             - no_news_for = (Time.now - vars[:updated_at]).round(0)
             = formatted_duration no_news_for
           - else
-            = "?"
+            = "?"

data/lib/rest-ftp-daemon/worker_pool.rb

@@ -32,7 +32,7 @@ module RestFtpDaemon
       if !(pools.is_a? Hash)
         log_error "create_threads: one JobWorker is the minimum (#{pools.inspect}"
       end
-      log_info "WorkerPool creating all workers with #{pools.to_hash.inspect}"
+      log_info "creating all workers with #{pools.to_hash.inspect}"
 
       # Start ConchitaWorker and ReporterWorker
       create_thread ConchitaWorker, :conchita
@@ -101,4 +101,4 @@ module RestFtpDaemon
     add_transaction_tracer :create_thread, category: :task
 
   end
-end
+end

data/lib/rest-ftp-daemon/workers/conchita.rb

@@ -41,4 +41,4 @@ module RestFtpDaemon
     end
 
   end
-end
+end

data/lib/rest-ftp-daemon/workers/reporter.rb

@@ -72,4 +72,4 @@ module RestFtpDaemon
     end
 
   end
-end
+end

data/lib/rest-ftp-daemon/workers/transfer.rb

@@ -15,7 +15,7 @@ module RestFtpDaemon
       return "invalid timeout" unless @config[:timeout].to_i > 0
 
       # Log that
-      log_info "JobWorker worker_init", {
+      log_info "worker_init", {
         pool: @pool,
         timeout: @config[:timeout]
       }
@@ -80,7 +80,7 @@ module RestFtpDaemon
       # If job status requires a retry, just restack it
       if !job.error
         # Processing successful
-        log_info "job_result: finished with success"
+        log_info "job_result: finished successfully"
         worker_status WORKER_STATUS_FINISHED, job
 
       elsif error_not_eligible(job)
@@ -141,4 +141,4 @@ module RestFtpDaemon
     end
 
   end
-end
+end