aspera-cli 4.10.0 → 4.12.0

data/lib/aspera/fasp/agent_direct.rb

@@ -16,14 +16,15 @@ require 'shellwords'
 module Aspera
   module Fasp
     # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
-    class AgentDirect < AgentBase
+    class AgentDirect < Aspera::Fasp::AgentBase
       # options for initialize (same as values in option transfer_info)
       DEFAULT_OPTIONS = {
         spawn_timeout_sec: 3,
         spawn_delay_sec:   2,
-        wss:               false,
+        wss:               true, # true: if both SSH and wss in ts: prefer wss
         multi_incr_udp:    true,
         resume:            {},
+        ascp_args:         [],
         quiet:             true # by default no interactive progress bar
       }.freeze
       private_constant :DEFAULT_OPTIONS
@@ -31,12 +32,8 @@ module Aspera
       # start ascp transfer (non blocking), single or multi-session
       # job information added to @jobs
       # @param transfer_spec [Hash] aspera transfer specification
-      # @param options [Hash] :resumer, :regenerate_token
-      def start_transfer(transfer_spec,options={})
-        raise 'option: must be hash (or nil)' unless options.is_a?(Hash)
-        job_options = options.clone
-        job_options[:resumer] ||= @resume_policy
-        job_options[:job_id] ||= SecureRandom.uuid
+      def start_transfer(transfer_spec, token_regenerator: nil)
+        the_job_id = SecureRandom.uuid
         # clone transfer spec because we modify it (first level keys)
         transfer_spec = transfer_spec.clone
         # if there is aspera tags
@@ -46,38 +43,38 @@ module Aspera
           # using a non unique id results in discard of tags in AoC, and a package is never finalized
           # all sessions in a multi-session transfer must have the same xfer_id (see admin manual)
           transfer_spec['tags']['aspera']['xfer_id'] ||= SecureRandom.uuid
-          Log.log.debug("xfer id=#{transfer_spec['xfer_id']}")
+          Log.log.debug{"xfer id=#{transfer_spec['xfer_id']}"}
           # TODO: useful ? node only ?
           transfer_spec['tags']['aspera']['xfer_retry'] ||= 3600
         end
-        Log.dump('ts',transfer_spec)
+        Log.dump('ts', transfer_spec)
 
         # add bypass keys when authentication is token and no auth is provided
-        if transfer_spec.has_key?('token') &&
-        !transfer_spec.has_key?('remote_password') &&
-        !transfer_spec.has_key?('EX_ssh_key_paths')
-          # transfer_spec['remote_password'] = Installation.instance.bypass_pass # not used
+        if transfer_spec.key?('token') &&
+            !transfer_spec.key?('remote_password') &&
+            !transfer_spec.key?('EX_ssh_key_paths')
+          # transfer_spec['remote_password'] = Installation.instance.bypass_pass # not used: no passphrase
           transfer_spec['EX_ssh_key_paths'] = Installation.instance.bypass_keys
         end
 
         # Compute this before using transfer spec because it potentially modifies the transfer spec
         # (even if the var is not used in single session)
         multi_session_info = nil
-        if transfer_spec.has_key?('multi_session')
+        if transfer_spec.key?('multi_session')
           multi_session_info = {
             count: transfer_spec['multi_session'].to_i
           }
           # Managed by multi-session, so delete from transfer spec
           transfer_spec.delete('multi_session')
           if multi_session_info[:count].negative?
-            Log.log.error("multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0")
+            Log.log.error{"multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0"}
             multi_session_info = nil
           elsif multi_session_info[:count].eql?(0)
-            Log.log.debug('multi_session count is zero: no multisession')
+            Log.log.debug('multi_session count is zero: no multi session')
             multi_session_info = nil
           elsif @options[:multi_incr_udp] # multi_session_info[:count] > 0
             # if option not true: keep default udp port for all sessions
-            multi_session_info[:udp_base] = transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] : TransferSpec::UDP_PORT
+            multi_session_info[:udp_base] = transfer_spec.key?('fasp_port') ? transfer_spec['fasp_port'] : TransferSpec::UDP_PORT
             # delete from original transfer spec, as we will increment values
             transfer_spec.delete('fasp_port')
             # override if specified, else use default value
@@ -85,30 +82,30 @@ module Aspera
         end
 
         # compute known args
-        env_args = Parameters.ts_to_env_args(transfer_spec,wss: @options[:wss])
+        env_args = Parameters.ts_to_env_args(transfer_spec, wss: @options[:wss], ascp_args: @options[:ascp_args])
 
         # add fallback cert and key as arguments if needed
         if %w[1 force].include?(transfer_spec['http_fallback'])
-          env_args[:args].unshift('-Y',Installation.instance.path(:fallback_key))
-          env_args[:args].unshift('-I',Installation.instance.path(:fallback_cert))
+          env_args[:args].unshift('-Y', Installation.instance.path(:fallback_key))
+          env_args[:args].unshift('-I', Installation.instance.path(:fallback_cert))
         end
 
         env_args[:args].unshift('-q') if @options[:quiet]
 
         # transfer job can be multi session
         xfer_job = {
-          id:       job_options[:job_id],
+          id:       the_job_id,
           sessions: [] # all sessions as below
         }
 
         # generic session information
         session = {
-          thread:   nil,         # Thread object monitoring management port, not nil when pushed to :sessions
-          error:    nil,         # exception if failed
-          io:       nil,         # management port server socket
-          id:       nil,         # SessionId from INIT message in mgt port
-          env_args: env_args,    # env vars and args to ascp (from transfer spec)
-          options:  job_options  # [Hash]
+          thread:            nil,               # Thread object monitoring management port, not nil when pushed to :sessions
+          error:             nil,               # exception if failed
+          io:                nil,               # management port server socket
+          id:                nil,               # SessionId from INIT message in mgt port
+          token_regenerator: token_regenerator, # regenerate bearer token with oauth
+          env_args:          env_args           # env vars and args to ascp (from transfer spec)
         }
 
         if multi_session_info.nil?
@@ -127,7 +124,7 @@ module Aspera
             this_session[:env_args][:args] = this_session[:env_args][:args].clone
             this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_info[:count]}")
             # option: increment (default as per ascp manual) or not (cluster on other side ?)
-            this_session[:env_args][:args].unshift('-O',(multi_session_info[:udp_base] + i - 1).to_s) if @options[:multi_incr_udp]
+            this_session[:env_args][:args].unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @options[:multi_incr_udp]
             this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
             xfer_job[:sessions].push(this_session)
           end
@@ -135,10 +132,10 @@ module Aspera
         Log.log.debug('started session thread(s)')
 
         # add job to list of jobs
-        @jobs[job_options[:job_id]] = xfer_job
-        Log.log.debug("jobs: #{@jobs.keys.count}")
+        @jobs[the_job_id] = xfer_job
+        Log.log.debug{"jobs: #{@jobs.keys.count}"}
 
-        return job_options[:job_id]
+        return the_job_id
       end # start_transfer
 
       # wait for completion of all jobs started
@@ -147,9 +144,9 @@ module Aspera
         Log.log.debug('wait_for_transfers_completion')
         # set to non-nil to exit loop
         result = []
-        @jobs.each do |_id,job|
+        @jobs.each do |_id, job|
           job[:sessions].each do |session|
-            Log.log.debug("join #{session[:thread]}")
+            Log.log.debug{"join #{session[:thread]}"}
             session[:thread].join
             result.push(session[:error] || :success)
           end
@@ -173,13 +170,13 @@ module Aspera
       # @param env_args a hash containing :args :env :ascp_version
       # @param session this session information
       # could be private method
-      def start_transfer_with_args_env(env_args,session)
+      def start_transfer_with_args_env(env_args, session)
         raise 'env_args must be Hash' unless env_args.is_a?(Hash)
         raise 'session must be Hash' unless session.is_a?(Hash)
         # by default we assume an exception will be raised (for ensure block)
         exception_raised = true
         begin
-          Log.log.debug("env_args=#{env_args.inspect}")
+          Log.log.debug{"env_args=#{env_args.inspect}"}
           # get location of ascp executable
           ascp_path = @mutex.synchronize do
             Fasp::Installation.instance.path(env_args[:ascp_version])
@@ -187,24 +184,24 @@ module Aspera
           # (optional) check it exists
           raise Fasp::Error, "no such file: #{ascp_path}" unless File.exist?(ascp_path)
           # open random local TCP port for listening for ascp management
-          mgt_sock = TCPServer.new('127.0.0.1',0)
+          mgt_sock = TCPServer.new('127.0.0.1', 0)
           # clone arguments as we eed to modify with mgt port
           ascp_arguments = env_args[:args].clone
           # add management port
           ascp_arguments.unshift('-M', mgt_sock.addr[1].to_s)
           # start ascp in sub process
           Log.log.debug do
-            'execute: '+
-            env_args[:env].map{|k,v| "#{k}=#{Shellwords.shellescape(v)}"}.join(' ')+
-            ' '+
-            Shellwords.shellescape(ascp_path)+
-            ' '+
-            ascp_arguments.map{|a|Shellwords.shellescape(a)}.join(' ')
+            'execute: ' +
+              env_args[:env].map{|k, v| "#{k}=#{Shellwords.shellescape(v)}"}.join(' ') +
+              ' ' +
+              Shellwords.shellescape(ascp_path) +
+              ' ' +
+              ascp_arguments.map{|a|Shellwords.shellescape(a)}.join(' ')
           end
           # start process
-          ascp_pid = Process.spawn(env_args[:env],[ascp_path,ascp_path],*ascp_arguments)
+          ascp_pid = Process.spawn(env_args[:env], [ascp_path, ascp_path], *ascp_arguments)
           # in parent, wait for connection to socket max 3 seconds
-          Log.log.debug("before accept for pid (#{ascp_pid})")
+          Log.log.debug{"before accept for pid (#{ascp_pid})"}
           # init management socket
           ascp_mgt_io = nil
           Timeout.timeout(@options[:spawn_timeout_sec]) do
@@ -214,7 +211,7 @@ module Aspera
             # TODO: use same value as Encoding.default_external
             ascp_mgt_io.set_encoding(Encoding::UTF_8)
           end
-          Log.log.debug("after accept (#{ascp_mgt_io})")
+          Log.log.debug{"after accept (#{ascp_mgt_io})"}
           session[:io] = ascp_mgt_io
           # exact text for event, with \n
           current_event_text = ''
@@ -230,7 +227,7 @@ module Aspera
             break if line.nil?
             current_event_text += line
             line.chomp!
-            Log.log.debug("line=[#{line}]")
+            Log.log.debug{"line=[#{line}]"}
             case line
             when 'FASPMGR 2'
               # begin event
@@ -243,12 +240,12 @@ module Aspera
               # empty line is separator to end event information
               raise 'unexpected empty line' if current_event_data.nil?
               current_event_data[AgentBase::LISTENER_SESSION_ID_B] = ascp_pid
-              notify_listeners(current_event_text,current_event_data)
+              notify_listeners(current_event_text, current_event_data)
               case current_event_data['Type']
               when 'INIT'
                 session[:id] = current_event_data['SessionId']
-                Log.log.debug("session id: #{session[:id]}")
-              when 'DONE','ERROR'
+                Log.log.debug{"session id: #{session[:id]}"}
+              when 'DONE', 'ERROR'
                 # TODO: check if this is always the last event
                 last_status_event = current_event_data
               end # event type
@@ -263,20 +260,20 @@ module Aspera
               # all went well
               exception_raised = false
             when 'ERROR'
-              Log.log.error("code: #{last_status_event['Code']}")
+              Log.log.error{"code: #{last_status_event['Code']}"}
               if /bearer token/i.match?(last_status_event['Description'])
                 Log.log.error('need to regenerate token'.red)
-                if session[:options].is_a?(Hash) && session[:options].has_key?(:regenerate_token)
+                if session[:token_regenerator].respond_to?(:refreshed_transfer_token)
                   # regenerate token here, expired, or error on it
                   # Note: in multi-session, each session will have a different one.
-                  env_args[:env]['ASPERA_SCP_TOKEN'] = session[:options][:regenerate_token].call(true)
+                  env_args[:env]['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
                 end
               end
               # cannot resolve address
-              #if last_status_event['Code'].to_i.eql?(14)
-              #  Log.log.warn("host: #{}")
-              #end
-              raise Fasp::Error.new(last_status_event['Description'],last_status_event['Code'].to_i)
+              # if last_status_event['Code'].to_i.eql?(14)
+              #  Log.log.warn{"host: #{}"}
+              # end
+              raise Fasp::Error.new(last_status_event['Description'], last_status_event['Code'].to_i)
             else # case
               raise "unexpected last event type: #{last_status_event['Type']}"
             end
@@ -316,19 +313,19 @@ module Aspera
       # @param data command on mgt port, examples:
       # {'type'=>'START','source'=>_path_,'destination'=>_path_}
       # {'type'=>'DONE'}
-      def send_command(job_id,session_index,data)
+      def send_command(job_id, session_index, data)
         job = @jobs[job_id]
         raise 'no such job' if job.nil?
         session = job[:sessions][session_index]
         raise 'no such session' if session.nil?
-        Log.log.debug("command: #{data}")
+        Log.log.debug{"command: #{data}"}
         # build command
-        command = data.
-          keys.
-          map{|k|"#{k.capitalize}: #{data[k]}"}.
-          unshift('FASPMGR 2').
-          push('','').
-          join("\n")
+        command = data
+          .keys
+          .map{|k|"#{k.capitalize}: #{data[k]}"}
+          .unshift('FASPMGR 2')
+          .push('', '')
+          .join("\n")
         session[:io].puts(command)
       end
 
@@ -345,12 +342,12 @@ module Aspera
         @options = DEFAULT_OPTIONS.dup
         if !options.nil?
           raise "expecting Hash (or nil), but have #{options.class}" unless options.is_a?(Hash)
-          options.each do |k,v|
-            raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.has_key?(k)
+          options.each do |k, v|
+            raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.key?(k)
             @options[k] = v
           end
         end
-        Log.log.debug("local options= #{options}")
+        Log.log.debug{"local options= #{options}"}
         @resume_policy = ResumePolicy.new(@options[:resume].symbolize_keys)
       end
 
@@ -360,17 +357,17 @@ module Aspera
         begin
           # set name for logging
           Thread.current[:name] = 'transfer'
-          Log.log.debug("ENTER (#{Thread.current[:name]})")
+          Log.log.debug{"ENTER (#{Thread.current[:name]})"}
           # start transfer with selected resumer policy
-          session[:options][:resumer].execute_with_resume do
-            start_transfer_with_args_env(session[:env_args],session)
+          @resume_policy.execute_with_resume do
+            start_transfer_with_args_env(session[:env_args], session)
           end
           Log.log.debug('transfer ok'.bg_green)
         rescue StandardError => e
           session[:error] = e
-          Log.log.error("Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red) if Log.instance.level.eql?(:debug)
+          Log.log.error{"Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red} if Log.instance.level.eql?(:debug)
         end
-        Log.log.debug("EXIT (#{Thread.current[:name]})")
+        Log.log.debug{"EXIT (#{Thread.current[:name]})"}
       end
     end # AgentDirect
   end

data/lib/aspera/fasp/agent_httpgw.rb

@@ -14,24 +14,25 @@ require 'json'
 module Aspera
   module Fasp
     # start a transfer using Aspera HTTP Gateway, using web socket session for uploads
-    class AgentHttpgw < AgentBase
+    class AgentHttpgw < Aspera::Fasp::AgentBase
       # message returned by HTTP GW in case of success
       MSG_END_UPLOAD = 'end upload'
       MSG_END_SLICE = 'end_slice_upload'
+      # options available in CLI (transfer_info)
       DEFAULT_OPTIONS = {
         url:                    nil,
-        upload_chunksize:       64_000,
+        upload_chunk_size:      64_000,
         upload_bar_refresh_sec: 0.5
       }.freeze
-      DEFAULT_BASE_PATH='/aspera/http-gwy'
+      DEFAULT_BASE_PATH = '/aspera/http-gwy'
       # upload endpoints
-      V1_UPLOAD='/v1/upload'
-      V2_UPLOAD='/v2/upload'
-      private_constant :DEFAULT_OPTIONS,:MSG_END_UPLOAD,:MSG_END_SLICE,:V1_UPLOAD,:V2_UPLOAD
+      V1_UPLOAD = '/v1/upload'
+      V2_UPLOAD = '/v2/upload'
+      private_constant :DEFAULT_OPTIONS, :MSG_END_UPLOAD, :MSG_END_SLICE, :V1_UPLOAD, :V2_UPLOAD
 
       # send message on http gw web socket
       def ws_snd_json(data)
-        @slice_uploads += 1 if data.has_key?(:slice_upload)
+        @slice_uploads += 1 if data.key?(:slice_upload)
         Log.log.debug{JSON.generate(data)}
         ws_send(JSON.generate(data))
       end
@@ -47,7 +48,7 @@ module Aspera
         # we need to keep track of actual file path because transfer spec is modified to be sent in web socket
         source_paths = []
         # get source root or nil
-        source_root = transfer_spec.has_key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
+        source_root = transfer_spec.key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
         # source root is ignored by GW, used only here
         transfer_spec.delete('source_root')
         # compute total size of files to upload (for progress)
@@ -56,7 +57,7 @@ module Aspera
           # save actual file location to be able read contents later
           full_src_filepath = item['source']
           # add source root if needed
-          full_src_filepath = File.join(source_root,full_src_filepath) unless source_root.nil?
+          full_src_filepath = File.join(source_root, full_src_filepath) unless source_root.nil?
           # GW expects a simple file name in 'source' but if user wants to change the name, we take it
           item['source'] = File.basename(item['destination'].nil? ? item['source'] : item['destination'])
           item['file_size'] = File.size(full_src_filepath)
@@ -66,18 +67,18 @@ module Aspera
         end
         # identify this session uniquely
         session_id = SecureRandom.uuid
-        @slice_uploads=0
+        @slice_uploads = 0
         # web socket endpoint: by default use v2 (newer gateways), without base64 encoding
         upload_api_version = V2_UPLOAD
         # is the latest supported? else revert to old api
-        upload_api_version=V1_UPLOAD unless @api_info['endpoints'].any?{|i|i.include?(upload_api_version)}
+        upload_api_version = V1_UPLOAD unless @api_info['endpoints'].any?{|i|i.include?(upload_api_version)}
         Log.log.debug{"api version: #{upload_api_version}"}
-        url=File.join(@gw_api.params[:base_url],upload_api_version)
-        #uri = URI.parse(url)
+        url = File.join(@gw_api.params[:base_url], upload_api_version)
+        # uri = URI.parse(url)
         # open web socket to end point (equivalent to Net::HTTP.start)
         http_socket = Rest.start_http_session(url)
         @ws_io = http_socket.instance_variable_get(:@socket)
-        #@ws_io.debug_output = Log.log
+        # @ws_io.debug_output = Log.log
         @ws_handshake = ::WebSocket::Handshake::Client.new(url: url, headers: {})
         @ws_io.write(@ws_handshake.to_s)
         sleep(0.1)
@@ -85,21 +86,21 @@ module Aspera
         raise 'Error in websocket handshake' unless @ws_handshake.finished?
         Log.log.debug('ws: handshake success')
         # data shared between main thread and read thread
-        shared_info={
+        shared_info = {
           read_exception: nil, # error message if any in callback
           end_uploads:    0 # number of files totally sent
-          #mutex: Mutex.new
-          #cond_var: ConditionVariable.new
+          # mutex: Mutex.new
+          # cond_var: ConditionVariable.new
         }
         # start read thread
         ws_read_thread = Thread.new do
           Log.log.debug('ws: thread: started')
           frame = ::WebSocket::Frame::Incoming::Client.new
           loop do
-            begin
+            begin # rubocop:disable Style/RedundantBegin
               frame << @ws_io.readuntil("\n")
               while (msg = frame.next)
-                Log.log.debug("ws: thread: message: #{msg.data} #{shared_info[:end_uploads]}")
+                Log.log.debug{"ws: thread: message: #{msg.data} #{shared_info[:end_uploads]}"}
                 message = msg.data
                 if message.eql?(MSG_END_UPLOAD)
                   shared_info[:end_uploads] += 1
@@ -122,12 +123,12 @@ module Aspera
               break
             end
           end
-          Log.log.debug("ws: thread: stopping #{shared_info[:read_exception]} #{shared_info[:read_exception].class}")
+          Log.log.debug{"ws: thread: stopping (exc=#{shared_info[:read_exception]},cls=#{shared_info[:read_exception].class})"}
         end
         # notify progress bar
-        notify_begin(session_id,total_size)
+        notify_begin(session_id, total_size)
         # first step send transfer spec
-        Log.dump(:ws_spec,transfer_spec)
+        Log.dump(:ws_spec, transfer_spec)
         ws_snd_json(transfer_spec: transfer_spec)
         # current file index
         file_index = 0
@@ -135,58 +136,63 @@ module Aspera
         sent_bytes = 0
         # last progress event
         last_progress_time = nil
-        begin
-          transfer_spec['paths'].each do |item|
-            # TODO: get mime type?
-            file_mime_type = ''
-            file_size = item['file_size']
-            file_name = File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
-            # compute total number of slices
-            numslices = 1 + ((file_size - 1) / @options[:upload_chunksize])
-            File.open(source_paths[file_index]) do |file|
-              # current slice index
-              slicenum = 0
-              while !file.eof?
-                # interrupt main thread if read thread failed
-                raise shared_info[:read_exception] unless shared_info[:read_exception].nil?
-                data = file.read(@options[:upload_chunksize])
-                slice_data = {
-                  name:         file_name,
-                  type:         file_mime_type,
-                  size:         file_size,
-                  slice:        slicenum,
-                  total_slices: numslices,
-                  fileIndex:    file_index
-                }
-                #Log.dump(:slice_data,slice_data) #if slicenum.eql?(0)
+
+        transfer_spec['paths'].each do |item|
+          # TODO: get mime type?
+          file_mime_type = ''
+          file_size = item['file_size']
+          file_name = File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
+          # compute total number of slices
+          slice_total = ((file_size - 1) / @options[:upload_chunk_size]) + 1
+          File.open(source_paths[file_index]) do |file|
+            # current slice index
+            slice_index = 0
+            until file.eof?
+              data = file.read(@options[:upload_chunk_size])
+              slice_data = {
+                name:         file_name,
+                type:         file_mime_type,
+                size:         file_size,
+                slice:        slice_index,
+                total_slices: slice_total,
+                fileIndex:    file_index
+              }
+              # Log.dump(:slice_data,slice_data) #if slice_index.eql?(0)
+              # interrupt main thread if read thread failed
+              raise shared_info[:read_exception] unless shared_info[:read_exception].nil?
+              begin
                 if upload_api_version.eql?(V1_UPLOAD)
                   slice_data[:data] = Base64.strict_encode64(data)
                   ws_snd_json(slice_upload: slice_data)
                 else
-                  ws_snd_json(slice_upload: slice_data) if slicenum.eql?(0)
-                  ws_send(data,type: :binary)
-                  Log.log.debug{"ws: sent buffer: #{file_index} / #{slicenum}"}
-                  ws_snd_json(slice_upload: slice_data) if slicenum.eql?(numslices-1)
-                end
-                sent_bytes += data.length
-                currenttime = Time.now
-                if last_progress_time.nil? || ((currenttime - last_progress_time) > @options[:upload_bar_refresh_sec])
-                  notify_progress(session_id,sent_bytes)
-                  last_progress_time = currenttime
+                  ws_snd_json(slice_upload: slice_data) if slice_index.eql?(0)
+                  ws_send(data, type: :binary)
+                  Log.log.debug{"ws: sent buffer: #{file_index} / #{slice_index}"}
+                  ws_snd_json(slice_upload: slice_data) if slice_index.eql?(slice_total - 1)
                 end
-                slicenum += 1
+              rescue Errno::EPIPE => e
+                raise shared_info[:read_exception] unless shared_info[:read_exception].nil?
+                raise e
+              end
+              sent_bytes += data.length
+              current_time = Time.now
+              if last_progress_time.nil? || ((current_time - last_progress_time) > @options[:upload_bar_refresh_sec])
+                notify_progress(session_id, sent_bytes)
+                last_progress_time = current_time
               end
+              slice_index += 1
             end
-            file_index += 1
           end
+          file_index += 1
         end
+
         Log.log.debug('Finished upload')
         ws_read_thread.join
         Log.log.debug{"result: #{shared_info[:end_uploads]} / #{@slice_uploads}"}
         ws_send(nil, type: :close) unless @ws_io.nil?
         @ws_io = nil
         http_socket&.finish
-        notify_progress(session_id,sent_bytes)
+        notify_progress(session_id, sent_bytes)
         notify_end(session_id)
       end
 
@@ -194,18 +200,18 @@ module Aspera
         transfer_spec['zip_required'] ||= false
         transfer_spec['source_root'] ||= '/'
         # is normally provided by application, like package name
-        if !transfer_spec.has_key?('download_name')
+        if !transfer_spec.key?('download_name')
           # by default it is the name of first file
-          dname = File.basename(transfer_spec['paths'].first['source'])
+          download_name = File.basename(transfer_spec['paths'].first['source'])
           # we remove extension
-          dname = dname.gsub(/\.@gw_api.*$/,'')
+          download_name = download_name.gsub(/\.@gw_api.*$/, '')
           # ands add indication of number of files if there is more than one
           if transfer_spec['paths'].length > 1
-            dname += " #{transfer_spec['paths'].length} Files"
+            download_name += " #{transfer_spec['paths'].length} Files"
           end
-          transfer_spec['download_name'] = dname
+          transfer_spec['download_name'] = download_name
         end
-        creation = @gw_api.create('v1/download',{'transfer_spec' => transfer_spec})[:data]
+        creation = @gw_api.create('v1/download', {'transfer_spec' => transfer_spec})[:data]
         transfer_uuid = creation['url'].split('/').last
         file_dest =
           if transfer_spec['zip_required'] || transfer_spec['paths'].length > 1
@@ -215,19 +221,18 @@ module Aspera
             # it is a plain file if we don't require zip and there is only one file
             File.basename(transfer_spec['paths'].first['source'])
           end
-        file_dest = File.join(transfer_spec['destination_root'],file_dest)
-        @gw_api.call({operation: 'GET',subpath: "v1/download/#{transfer_uuid}",save_to_file: file_dest})
+        file_dest = File.join(transfer_spec['destination_root'], file_dest)
+        @gw_api.call({operation: 'GET', subpath: "v1/download/#{transfer_uuid}", save_to_file: file_dest})
       end
 
       # start FASP transfer based on transfer spec (hash table)
       # note that it is asynchronous
       # HTTP download only supports file list
-      def start_transfer(transfer_spec,options={})
+      def start_transfer(transfer_spec, token_regenerator: nil)
         raise 'GW URL must be set' if @gw_api.nil?
-        raise 'option: must be hash (or nil)' unless options.is_a?(Hash)
         raise 'paths: must be Array' unless transfer_spec['paths'].is_a?(Array)
         raise 'only token based transfer is supported in GW' unless transfer_spec['token'].is_a?(String)
-        Log.dump(:user_spec,transfer_spec)
+        Log.dump(:user_spec, transfer_spec)
         transfer_spec['authentication'] ||= 'token'
         case transfer_spec['direction']
         when Fasp::TransferSpec::DIRECTION_SEND
@@ -253,17 +258,17 @@ module Aspera
       private
 
       def initialize(opts)
-        Log.log.debug("local options= #{opts}")
+        Log.log.debug{"local options= #{opts}"}
         # set default options and override if specified
         @options = DEFAULT_OPTIONS.dup
         raise "httpgw agent parameters (transfer_info): expecting Hash, but have #{opts.class}" unless opts.is_a?(Hash)
-        opts.symbolize_keys.each do |k,v|
-          raise "httpgw agent parameter: Unknown: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.has_key?(k)
+        opts.symbolize_keys.each do |k, v|
+          raise "httpgw agent parameter: Unknown: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.key?(k)
           @options[k] = v
         end
         raise 'missing param: url' if @options[:url].nil?
         # remove /v1 from end
-        @options[:url].gsub(%r{/v1/*$},'')
+        @options[:url].gsub(%r{/v1/*$}, '')
         super()
         @gw_api = Rest.new({base_url: @options[:url]})
         @api_info = @gw_api.read('v1/info')[:data]