aspera-cli 4.7.0 → 4.9.0

data/lib/aspera/fasp/agent_httpgw.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'aspera/fasp/agent_base'
 require 'aspera/fasp/transfer_spec'
 require 'aspera/log'
@@ -15,9 +16,9 @@ module Aspera
     # start a transfer using Aspera HTTP Gateway, using web socket session
     class AgentHttpgw < AgentBase
       # message returned by HTTP GW in case of success
-      OK_MESSAGE='end upload'
+      OK_MESSAGE = 'end upload'
       # refresh rate for progress
-      UPLOAD_REFRESH_SEC=0.5
+      UPLOAD_REFRESH_SEC = 0.5
       private_constant :OK_MESSAGE,:UPLOAD_REFRESH_SEC
       # send message on http gw web socket
       def ws_send(ws,type,data)
@@ -26,9 +27,9 @@ module Aspera
 
       def upload(transfer_spec)
         # total size of all files
-        total_size=0
+        total_size = 0
         # we need to keep track of actual file path because transfer spec is modified to be sent in web socket
-        source_paths=[]
+        source_paths = []
         # get source root or nil
         source_root = transfer_spec.has_key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
         # source root is ignored by GW, used only here
@@ -37,41 +38,46 @@ module Aspera
         # modify transfer spec to be suitable for GW
         transfer_spec['paths'].each do |item|
           # save actual file location to be able read contents later
-          full_src_filepath=item['source']
+          full_src_filepath = item['source']
           # add source root if needed
-          full_src_filepath=File.join(source_root,full_src_filepath) unless source_root.nil?
+          full_src_filepath = File.join(source_root,full_src_filepath) unless source_root.nil?
           # GW expects a simple file name in 'source' but if user wants to change the name, we take it
-          item['source']=File.basename(item['destination'].nil? ? item['source'] : item['destination'])
-          item['file_size']=File.size(full_src_filepath)
-          total_size+=item['file_size']
+          item['source'] = File.basename(item['destination'].nil? ? item['source'] : item['destination'])
+          item['file_size'] = File.size(full_src_filepath)
+          total_size += item['file_size']
           # save so that we can actually read the file later
           source_paths.push(full_src_filepath)
         end
 
-        session_id=SecureRandom.uuid
-        ws=::WebSocket::Client::Simple::Client.new
+        session_id = SecureRandom.uuid
+        ws = ::WebSocket::Client::Simple::Client.new
         # error message if any in callback
-        error=nil
+        error = nil
         # number of files totally sent
-        received=0
+        received = 0
         # setup callbacks on websocket
-        ws.on :message do |msg|
+        ws.on(:message) do |msg|
           Log.log.info("ws: message: #{msg.data}")
-          message=msg.data
+          message = msg.data
           if message.eql?(OK_MESSAGE)
-            received+=1
+            received += 1
           else
             message.chomp!
-            error=message.start_with?('"') && message.end_with?('"') ? JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message'] : "expecting quotes in [#{message}]"
+            error =
+              if message.start_with?('"') && message.end_with?('"')
+                JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
+              else
+                "expecting quotes in [#{message}]"
+              end
           end
         end
-        ws.on :error do |e|
-          error=e
+        ws.on(:error) do |e|
+          error = e
         end
-        ws.on :open do
+        ws.on(:open) do
           Log.log.info('ws: open')
         end
-        ws.on :close do
+        ws.on(:close) do
           Log.log.info('ws: close')
         end
         # open web socket to end point
@@ -87,76 +93,77 @@ module Aspera
         Log.dump(:ws_spec,transfer_spec)
         ws_send(ws,:transfer_spec,transfer_spec)
         # current file index
-        file_index=0
+        file_index = 0
         # aggregate size sent
-        sent_bytes=0
+        sent_bytes = 0
         # last progress event
-        lastevent=nil
+        lastevent = nil
         transfer_spec['paths'].each do |item|
           # TODO: get mime type?
-          file_mime_type=''
-          file_size=item['file_size']
-          file_name=File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
+          file_mime_type = ''
+          file_size = item['file_size']
+          file_name = File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
           # compute total number of slices
-          numslices=1+((file_size-1)/@upload_chunksize)
+          numslices = 1 + ((file_size - 1) / @upload_chunksize)
           File.open(source_paths[file_index]) do |file|
             # current slice index
-            slicenum=0
+            slicenum = 0
             while !file.eof?
-              data=file.read(@upload_chunksize)
-              slice_data={
-                name: file_name,
-                type: file_mime_type,
-                size: file_size,
-                data: Base64.strict_encode64(data),
-                slice: slicenum,
+              data = file.read(@upload_chunksize)
+              slice_data = {
+                name:         file_name,
+                type:         file_mime_type,
+                size:         file_size,
+                data:         Base64.strict_encode64(data),
+                slice:        slicenum,
                 total_slices: numslices,
-                fileIndex: file_index
+                fileIndex:    file_index
               }
               # log without data
-              Log.dump(:slide_data,slice_data.keys.each_with_object({}){|i,m|m[i]=i.eql?(:data)?'base64 data':slice_data[i];}) if slicenum.eql?(0)
+              Log.dump(:slide_data,slice_data.keys.each_with_object({}){|i,m|m[i] = i.eql?(:data) ? 'base64 data' : slice_data[i];}) if slicenum.eql?(0)
               ws_send(ws,:slice_upload, slice_data)
-              sent_bytes+=data.length
-              currenttime=Time.now
-              if lastevent.nil? || ((currenttime-lastevent)>UPLOAD_REFRESH_SEC)
+              sent_bytes += data.length
+              currenttime = Time.now
+              if lastevent.nil? || ((currenttime - lastevent) > UPLOAD_REFRESH_SEC)
                 notify_progress(session_id,sent_bytes)
-                lastevent=currenttime
+                lastevent = currenttime
               end
-              slicenum+=1
+              slicenum += 1
               raise error unless error.nil?
             end
           end
-          file_index+=1
+          file_index += 1
         end
         ws.close
         notify_end(session_id)
       end
 
       def download(transfer_spec)
-        transfer_spec['zip_required']||=false
-        transfer_spec['source_root']||='/'
+        transfer_spec['zip_required'] ||= false
+        transfer_spec['source_root'] ||= '/'
         # is normally provided by application, like package name
         if !transfer_spec.has_key?('download_name')
           # by default it is the name of first file
-          dname=File.basename(transfer_spec['paths'].first['source'])
+          dname = File.basename(transfer_spec['paths'].first['source'])
           # we remove extension
-          dname=dname.gsub(/\.@gw_api.*$/,'')
+          dname = dname.gsub(/\.@gw_api.*$/,'')
           # ands add indication of number of files if there is more than one
           if transfer_spec['paths'].length > 1
-            dname+=" #{transfer_spec['paths'].length} Files"
+            dname += " #{transfer_spec['paths'].length} Files"
           end
-          transfer_spec['download_name']=dname
-        end
-        creation=@gw_api.create('download',{'transfer_spec'=>transfer_spec})[:data]
-        transfer_uuid=creation['url'].split('/').last
-        file_dest=if transfer_spec['zip_required'] || transfer_spec['paths'].length > 1
-                    # it is a zip file if zip is required or there is more than 1 file
-                    transfer_spec['download_name']+'.zip'
-                  else
-                    # it is a plain file if we don't require zip and there is only one file
-                    File.basename(transfer_spec['paths'].first['source'])
+          transfer_spec['download_name'] = dname
         end
-        file_dest=File.join(transfer_spec['destination_root'],file_dest)
+        creation = @gw_api.create('download',{'transfer_spec' => transfer_spec})[:data]
+        transfer_uuid = creation['url'].split('/').last
+        file_dest =
+          if transfer_spec['zip_required'] || transfer_spec['paths'].length > 1
+            # it is a zip file if zip is required or there is more than 1 file
+            transfer_spec['download_name'] + '.zip'
+          else
+            # it is a plain file if we don't require zip and there is only one file
+            File.basename(transfer_spec['paths'].first['source'])
+          end
+        file_dest = File.join(transfer_spec['destination_root'],file_dest)
         @gw_api.call({operation: 'GET',subpath: "download/#{transfer_uuid}",save_to_file: file_dest})
       end
 
@@ -169,7 +176,7 @@ module Aspera
         raise 'paths: must be Array' unless transfer_spec['paths'].is_a?(Array)
         raise 'only token based transfer is supported in GW' unless transfer_spec['token'].is_a?(String)
         Log.dump(:user_spec,transfer_spec)
-        transfer_spec['authentication']||='token'
+        transfer_spec['authentication'] ||= 'token'
         case transfer_spec['direction']
         when Fasp::TransferSpec::DIRECTION_SEND
           upload(transfer_spec)
@@ -195,13 +202,13 @@ module Aspera
 
       def initialize(params)
         raise 'params must be Hash' unless params.is_a?(Hash)
-        params=params.symbolize_keys
+        params = params.symbolize_keys
         raise 'must have only one param: url' unless params.keys.eql?([:url])
         super()
-        @gw_api=Rest.new({base_url: params[:url]})
+        @gw_api = Rest.new({base_url: params[:url]})
         api_info = @gw_api.read('info')[:data]
         Log.log.info(api_info.to_s)
-        @upload_chunksize=128_000 # TODO: configurable ?
+        @upload_chunksize = 128_000 # TODO: configurable ?
       end
     end # AgentHttpgw
   end

data/lib/aspera/fasp/agent_node.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'aspera/fasp/agent_base'
 require 'aspera/fasp/transfer_spec'
 require 'aspera/log'
@@ -13,27 +14,27 @@ module Aspera
       attr_writer :options
       def initialize(options)
         raise 'node specification must be Hash' unless options.is_a?(Hash)
-        [:url,:username,:password].each { |k| raise "missing parameter [#{k}] in node specification: #{options}" unless options.has_key?(k) }
+        %i[url username password].each { |k| raise "missing parameter [#{k}] in node specification: #{options}" unless options.has_key?(k) }
         super()
         # root id is required for access key
-        @root_id=options[:root_id]
-        rest_params={ base_url: options[:url]}
-        if options[:password].match(/^Bearer /)
-          rest_params[:headers]={
-            'X-Aspera-AccessKey'=>options[:username],
-            'Authorization'     =>options[:password]
+        @root_id = options[:root_id]
+        rest_params = { base_url: options[:url]}
+        if /^Bearer /.match?(options[:password])
+          rest_params[:headers] = {
+            'X-Aspera-AccessKey' => options[:username],
+            'Authorization'      => options[:password]
           }
           raise 'root_id is required for access key' if @root_id.nil?
         else
-          rest_params[:auth]={
+          rest_params[:auth] = {
             type:     :basic,
             username: options[:username],
             password: options[:password]
           }
         end
-        @node_api=Rest.new(rest_params)
+        @node_api = Rest.new(rest_params)
         # TODO: currently only supports one transfer. This is bad shortcut. but ok for CLI.
-        @transfer_id=nil
+        @transfer_id = nil
       end
 
       # used internally to ensure node api is set before using.
@@ -49,7 +50,7 @@ module Aspera
         if !@node_api.nil? && !new_value.nil?
           Log.log.warn('overriding existing node api value')
         end
-        @node_api=new_value
+        @node_api = new_value
       end
 
       # generic method
@@ -57,8 +58,8 @@ module Aspera
         # add root id if access key
         if !@root_id.nil?
           case transfer_spec['direction']
-          when Fasp::TransferSpec::DIRECTION_SEND then transfer_spec['source_root_id']=@root_id
-          when Fasp::TransferSpec::DIRECTION_RECEIVE then transfer_spec['destination_root_id']=@root_id
+          when Fasp::TransferSpec::DIRECTION_SEND then transfer_spec['source_root_id'] = @root_id
+          when Fasp::TransferSpec::DIRECTION_RECEIVE then transfer_spec['destination_root_id'] = @root_id
           else raise "unexpected direction in ts: #{transfer_spec['direction']}"
           end
         end
@@ -69,31 +70,31 @@ module Aspera
             Log.log.warn('Both ssh_private_key and EX_ssh_key_paths are present, using ssh_private_key')
           else
             Log.log.warn('EX_ssh_key_paths has multiple keys, using first one only') unless transfer_spec['EX_ssh_key_paths'].length.eql?(1)
-            transfer_spec['ssh_private_key']=File.read(transfer_spec['EX_ssh_key_paths'].first)
+            transfer_spec['ssh_private_key'] = File.read(transfer_spec['EX_ssh_key_paths'].first)
             transfer_spec.delete('EX_ssh_key_paths')
           end
         end
         if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags']['aspera'].is_a?(Hash)
-          transfer_spec['tags']['aspera']['xfer_retry']||=150
+          transfer_spec['tags']['aspera']['xfer_retry'] ||= 150
         end
         # optimisation in case of sending to the same node
         if transfer_spec['remote_host'].eql?(URI.parse(node_api_.params[:base_url]).host)
-          transfer_spec['remote_host']='localhost'
+          transfer_spec['remote_host'] = 'localhost'
         end
-        resp=node_api_.create('ops/transfers',transfer_spec)[:data]
-        @transfer_id=resp['id']
+        resp = node_api_.create('ops/transfers',transfer_spec)[:data]
+        @transfer_id = resp['id']
         Log.log.debug("tr_id=#{@transfer_id}")
         return @transfer_id
       end
 
       # generic method
       def wait_for_transfers_completion
-        started=false
-        spinner=nil
+        started = false
+        spinner = nil
         # lets emulate management events to display progress bar
         loop do
           # status is empty sometimes with status 200...
-          trdata=node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {'status'=>'unknown'} rescue {'status'=>'waiting(read error)'}
+          trdata = node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {'status' => 'unknown'} rescue {'status' => 'waiting(read error)'}
           case trdata['status']
           when 'completed'
             notify_end(@transfer_id)
@@ -111,7 +112,7 @@ module Aspera
             if !started && trdata['precalc'].is_a?(Hash) &&
             trdata['precalc']['status'].eql?('ready')
               notify_begin(@transfer_id,trdata['precalc']['bytes_expected'])
-              started=true
+              started = true
             else
               notify_progress(@transfer_id,trdata['bytes_transferred'])
             end
@@ -119,7 +120,7 @@ module Aspera
             Log.log.warn("trdata -> #{trdata}")
             raise Fasp::Error, "#{trdata['status']}: #{trdata['error_desc']}"
           end
-          sleep 1
+          sleep(1)
         end
         #TODO get status of sessions
         return []

data/lib/aspera/fasp/agent_trsdk.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'aspera/fasp/agent_base'
 require 'aspera/fasp/installation'
 require 'json'
@@ -9,19 +10,17 @@ module Aspera
       DEFAULT_OPTIONS = {
         address: '127.0.0.1',
         port:    55_002
-      }
+      }.freeze
       private_constant :DEFAULT_OPTIONS
 
       # options come from transfer_info
       def initialize(user_opts)
         raise "expecting Hash (or nil), but have #{user_opts.class}" unless user_opts.nil? || user_opts.is_a?(Hash)
         # set default options and override if specified
-        options=DEFAULT_OPTIONS.clone
-        if !user_opts.nil?
-          user_opts.each do |k,v|
-            raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.has_key?(k)
-            options[k]=v
-          end
+        options = DEFAULT_OPTIONS.dup
+        user_opts&.each do |k,v|
+          raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.has_key?(k)
+          options[k] = v
         end
         Log.log.debug("options= #{options}")
         super()
@@ -35,20 +34,20 @@ module Aspera
         rescue GRPC::Unavailable
           Log.log.warn('no daemon present, starting daemon...')
           # location of daemon binary
-          bin_folder=File.realpath(File.join(Installation.instance.sdk_ruby_folder,'..'))
+          bin_folder = File.realpath(File.join(Installation.instance.sdk_ruby_folder,'..'))
           # config file and logs are created in same folder
           conf_file = File.join(bin_folder,'sdk.conf')
           log_base = File.join(bin_folder,'transferd')
           # create a config file for daemon
           config = {
-            address: options[:address],
-            port:    options[:port],
+            address:      options[:address],
+            port:         options[:port],
             fasp_runtime: {
-            use_embedded: false,
-            user_defined: {
-            bin: bin_folder,
-            etc: bin_folder
-            }
+              use_embedded: false,
+              user_defined: {
+                bin: bin_folder,
+                etc: bin_folder
+              }
             }
           }
           File.write(conf_file,config.to_json)
@@ -62,9 +61,9 @@ module Aspera
       def start_transfer(transfer_spec,_options=nil)
         # create a transfer request
         transfer_request = Transfersdk::TransferRequest.new(
-        transferType: Transfersdk::TransferType::FILE_REGULAR, # transfer type (file/stream)
-        config: Transfersdk::TransferConfig.new, # transfer configuration
-        transferSpec: transfer_spec.to_json) # transfer definition
+          transferType: Transfersdk::TransferType::FILE_REGULAR, # transfer type (file/stream)
+          config: Transfersdk::TransferConfig.new, # transfer configuration
+          transferSpec: transfer_spec.to_json) # transfer definition
         # send start transfer request to the transfer manager daemon
         start_transfer_response = @transfer_client.start_transfer(transfer_request)
         Log.log.debug("start transfer response #{start_transfer_response}")
@@ -73,7 +72,7 @@ module Aspera
       end
 
       def wait_for_transfers_completion
-        started=false
+        started = false
         # monitor transfer status
         @transfer_client.monitor_transfers(Transfersdk::RegistrationRequest.new(transferId: [@transfer_id])) do |response|
           Log.dump(:response, response.to_h)
@@ -82,7 +81,7 @@ module Aspera
           when :RUNNING
             if !started && !response.sessionInfo.preTransferBytes.eql?(0)
               notify_begin(@transfer_id,response.sessionInfo.preTransferBytes)
-              started=true
+              started = true
             elsif started
               notify_progress(@transfer_id,response.transferInfo.bytesTransferred)
             end

data/lib/aspera/fasp/error.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'aspera/fasp/error_info'
 
 module Aspera
@@ -12,7 +13,7 @@ module Aspera
       end
 
       def info
-        r=Fasp::ERROR_INFO[@err_code] || {r: false, c: 'UNKNOWN', m: 'unknown', a: 'unknown'}
+        r = Fasp::ERROR_INFO[@err_code] || {r: false, c: 'UNKNOWN', m: 'unknown', a: 'unknown'}
         return r.merge({i: @err_code})
       end