aspera-cli 4.5.0 → 4.8.0

data/lib/aspera/fasp/agent_httpgw.rb

@@ -1,5 +1,7 @@
-#!/bin/echo this is a ruby class:
+# frozen_string_literal: true
+
 require 'aspera/fasp/agent_base'
+require 'aspera/fasp/transfer_spec'
 require 'aspera/log'
 require 'aspera/rest'
 require 'websocket-client-simple'
@@ -14,9 +16,9 @@ module Aspera
     # start a transfer using Aspera HTTP Gateway, using web socket session
     class AgentHttpgw < AgentBase
       # message returned by HTTP GW in case of success
-      OK_MESSAGE='end upload'
+      OK_MESSAGE = 'end upload'
       # refresh rate for progress
-      UPLOAD_REFRESH_SEC=0.5
+      UPLOAD_REFRESH_SEC = 0.5
       private_constant :OK_MESSAGE,:UPLOAD_REFRESH_SEC
       # send message on http gw web socket
       def ws_send(ws,type,data)
@@ -25,63 +27,64 @@ module Aspera
 
       def upload(transfer_spec)
         # total size of all files
-        total_size=0
+        total_size = 0
         # we need to keep track of actual file path because transfer spec is modified to be sent in web socket
-        source_paths=[]
+        source_paths = []
         # get source root or nil
-        source_root = (transfer_spec.has_key?('source_root') and !transfer_spec['source_root'].empty?) ? transfer_spec['source_root'] : nil
+        source_root = transfer_spec.has_key?('source_root') && !transfer_spec['source_root'].empty? ? transfer_spec['source_root'] : nil
         # source root is ignored by GW, used only here
         transfer_spec.delete('source_root')
         # compute total size of files to upload (for progress)
         # modify transfer spec to be suitable for GW
         transfer_spec['paths'].each do |item|
           # save actual file location to be able read contents later
-          full_src_filepath=item['source']
+          full_src_filepath = item['source']
           # add source root if needed
-          full_src_filepath=File.join(source_root,full_src_filepath) unless source_root.nil?
+          full_src_filepath = File.join(source_root,full_src_filepath) unless source_root.nil?
           # GW expects a simple file name in 'source' but if user wants to change the name, we take it
-          item['source']=File.basename(item['destination'].nil? ? item['source'] : item['destination'])
-          item['file_size']=File.size(full_src_filepath)
-          total_size+=item['file_size']
+          item['source'] = File.basename(item['destination'].nil? ? item['source'] : item['destination'])
+          item['file_size'] = File.size(full_src_filepath)
+          total_size += item['file_size']
           # save so that we can actually read the file later
           source_paths.push(full_src_filepath)
         end
 
-        session_id=SecureRandom.uuid
-        ws=::WebSocket::Client::Simple::Client.new
+        session_id = SecureRandom.uuid
+        ws = ::WebSocket::Client::Simple::Client.new
         # error message if any in callback
-        error=nil
+        error = nil
         # number of files totally sent
-        received=0
+        received = 0
         # setup callbacks on websocket
-        ws.on :message do |msg|
+        ws.on(:message) do |msg|
           Log.log.info("ws: message: #{msg.data}")
-          message=msg.data
+          message = msg.data
           if message.eql?(OK_MESSAGE)
-            received+=1
+            received += 1
           else
             message.chomp!
-            if message[0].eql?('"') and message[-1].eql?('"')
-              error=JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
-            else
-              error="expecting quotes in [#{message}]"
-            end
+            error =
+              if message.start_with?('"') && message.end_with?('"')
+                JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
+              else
+                "expecting quotes in [#{message}]"
+              end
           end
         end
-        ws.on :error do |e|
-          error=e
+        ws.on(:error) do |e|
+          error = e
         end
-        ws.on :open do
-          Log.log.info("ws: open")
+        ws.on(:open) do
+          Log.log.info('ws: open')
         end
-        ws.on :close do
-          Log.log.info("ws: close")
+        ws.on(:close) do
+          Log.log.info('ws: close')
         end
         # open web socket to end point
         ws.connect("#{@gw_api.params[:base_url]}/upload")
         # async wait ready
-        while !ws.open? and error.nil? do
-          Log.log.info("ws: wait")
+        while !ws.open? && error.nil?
+          Log.log.info('ws: wait')
           sleep(0.2)
         end
         # notify progress bar
@@ -90,93 +93,94 @@ module Aspera
         Log.dump(:ws_spec,transfer_spec)
         ws_send(ws,:transfer_spec,transfer_spec)
         # current file index
-        file_index=0
+        file_index = 0
         # aggregate size sent
-        sent_bytes=0
+        sent_bytes = 0
         # last progress event
-        lastevent=nil
+        lastevent = nil
         transfer_spec['paths'].each do |item|
           # TODO: get mime type?
-          file_mime_type=''
-          file_size=item['file_size']
-          file_name=File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
+          file_mime_type = ''
+          file_size = item['file_size']
+          file_name = File.basename(item[item['destination'].nil? ? 'source' : 'destination'])
           # compute total number of slices
-          numslices=1+(file_size-1)/@upload_chunksize
+          numslices = 1 + ((file_size - 1) / @upload_chunksize)
           File.open(source_paths[file_index]) do |file|
             # current slice index
-            slicenum=0
-            while !file.eof? do
-              data=file.read(@upload_chunksize)
-              slice_data={
-                name: file_name,
-                type: file_mime_type,
-                size: file_size,
-                data: Base64.strict_encode64(data),
-                slice: slicenum,
+            slicenum = 0
+            while !file.eof?
+              data = file.read(@upload_chunksize)
+              slice_data = {
+                name:         file_name,
+                type:         file_mime_type,
+                size:         file_size,
+                data:         Base64.strict_encode64(data),
+                slice:        slicenum,
                 total_slices: numslices,
-                fileIndex: file_index
+                fileIndex:    file_index
               }
               # log without data
-              Log.dump(:slide_data,slice_data.keys.inject({}){|m,i|m[i]=i.eql?(:data)?'base64 data':slice_data[i];m}) if slicenum.eql?(0)
+              Log.dump(:slide_data,slice_data.keys.each_with_object({}){|i,m|m[i] = i.eql?(:data) ? 'base64 data' : slice_data[i];}) if slicenum.eql?(0)
               ws_send(ws,:slice_upload, slice_data)
-              sent_bytes+=data.length
-              currenttime=Time.now
-              if lastevent.nil? or (currenttime-lastevent)>UPLOAD_REFRESH_SEC
+              sent_bytes += data.length
+              currenttime = Time.now
+              if lastevent.nil? || ((currenttime - lastevent) > UPLOAD_REFRESH_SEC)
                 notify_progress(session_id,sent_bytes)
-                lastevent=currenttime
+                lastevent = currenttime
               end
-              slicenum+=1
+              slicenum += 1
               raise error unless error.nil?
             end
           end
-          file_index+=1
+          file_index += 1
         end
         ws.close
         notify_end(session_id)
       end
 
       def download(transfer_spec)
-        transfer_spec['zip_required']||=false
-        transfer_spec['source_root']||='/'
+        transfer_spec['zip_required'] ||= false
+        transfer_spec['source_root'] ||= '/'
         # is normally provided by application, like package name
         if !transfer_spec.has_key?('download_name')
           # by default it is the name of first file
-          dname=File.basename(transfer_spec['paths'].first['source'])
+          dname = File.basename(transfer_spec['paths'].first['source'])
           # we remove extension
-          dname=dname.gsub(/\.@gw_api.*$/,'')
+          dname = dname.gsub(/\.@gw_api.*$/,'')
           # ands add indication of number of files if there is more than one
           if transfer_spec['paths'].length > 1
-            dname=dname+" #{transfer_spec['paths'].length} Files"
+            dname += " #{transfer_spec['paths'].length} Files"
           end
-          transfer_spec['download_name']=dname
-        end
-        creation=@gw_api.create('download',{'transfer_spec'=>transfer_spec})[:data]
-        transfer_uuid=creation['url'].split('/').last
-        if transfer_spec['zip_required'] or transfer_spec['paths'].length > 1
-          # it is a zip file if zip is required or there is more than 1 file
-          file_dest=transfer_spec['download_name']+'.zip'
-        else
-          # it is a plain file if we don't require zip and there is only one file
-          file_dest=File.basename(transfer_spec['paths'].first['source'])
+          transfer_spec['download_name'] = dname
         end
-        file_dest=File.join(transfer_spec['destination_root'],file_dest)
-        @gw_api.call({:operation=>'GET',:subpath=>"download/#{transfer_uuid}",:save_to_file=>file_dest})
+        creation = @gw_api.create('download',{'transfer_spec' => transfer_spec})[:data]
+        transfer_uuid = creation['url'].split('/').last
+        file_dest =
+          if transfer_spec['zip_required'] || transfer_spec['paths'].length > 1
+            # it is a zip file if zip is required or there is more than 1 file
+            transfer_spec['download_name'] + '.zip'
+          else
+            # it is a plain file if we don't require zip and there is only one file
+            File.basename(transfer_spec['paths'].first['source'])
+          end
+        file_dest = File.join(transfer_spec['destination_root'],file_dest)
+        @gw_api.call({operation: 'GET',subpath: "download/#{transfer_uuid}",save_to_file: file_dest})
       end
 
       # start FASP transfer based on transfer spec (hash table)
       # note that it is asynchronous
       # HTTP download only supports file list
       def start_transfer(transfer_spec,options={})
-        raise "GW URL must be set" unless !@gw_api.nil?
-        raise "option: must be hash (or nil)" unless options.is_a?(Hash)
-        raise "paths: must be Array" unless transfer_spec['paths'].is_a?(Array)
-        raise "only token based transfer is supported in GW" unless transfer_spec['token'].is_a?(String)
+        raise 'GW URL must be set' if @gw_api.nil?
+        raise 'option: must be hash (or nil)' unless options.is_a?(Hash)
+        raise 'paths: must be Array' unless transfer_spec['paths'].is_a?(Array)
+        raise 'only token based transfer is supported in GW' unless transfer_spec['token'].is_a?(String)
         Log.dump(:user_spec,transfer_spec)
-        transfer_spec['authentication']||='token'
+        transfer_spec['authentication'] ||= 'token'
         case transfer_spec['direction']
-        when 'send'
+        when Fasp::TransferSpec::DIRECTION_SEND
           upload(transfer_spec)
-        when 'receive'
+        when Fasp::TransferSpec::DIRECTION_RECEIVE
           download(transfer_spec)
         else
           raise "unexpected direction: [#{transfer_spec['direction']}]"
@@ -190,25 +194,22 @@ module Aspera
       end
 
       # terminates monitor thread
-      def shutdown
-      end
+      def shutdown; end
 
-      def url=(api_url)
-      end
+      def url=(api_url); end
 
       private
 
       def initialize(params)
-        raise "params must be Hash" unless params.is_a?(Hash)
-        params=params.symbolize_keys
-        raise "must have only one param: url" unless params.keys.eql?([:url])
+        raise 'params must be Hash' unless params.is_a?(Hash)
+        params = params.symbolize_keys
+        raise 'must have only one param: url' unless params.keys.eql?([:url])
         super()
-        @gw_api=Rest.new({:base_url => params[:url]})
+        @gw_api = Rest.new({base_url: params[:url]})
         api_info = @gw_api.read('info')[:data]
-        Log.log.info("#{api_info}")
-        @upload_chunksize=128000 # TODO: configurable ?
+        Log.log.info(api_info.to_s)
+        @upload_chunksize = 128_000 # TODO: configurable ?
       end
-
     end # AgentHttpgw
   end
 end

data/lib/aspera/fasp/agent_node.rb

@@ -1,4 +1,7 @@
+# frozen_string_literal: true
+
 require 'aspera/fasp/agent_base'
+require 'aspera/fasp/transfer_spec'
 require 'aspera/log'
 require 'tty-spinner'
 
@@ -10,28 +13,28 @@ module Aspera
       # option include: root_id if the node is an access key
       attr_writer :options
       def initialize(options)
-        raise "node specification must be Hash" unless options.is_a?(Hash)
-        [:url,:username,:password].each { |k| raise "missing parameter [#{k}] in node specification: #{options}" unless options.has_key?(k) }
+        raise 'node specification must be Hash' unless options.is_a?(Hash)
+        %i[url username password].each { |k| raise "missing parameter [#{k}] in node specification: #{options}" unless options.has_key?(k) }
         super()
         # root id is required for access key
-        @root_id=options[:root_id]
-        rest_params={ base_url: options[:url]}
-        if options[:password].match(/^Bearer /)
-          rest_params[:headers]={
-            'X-Aspera-AccessKey'=>options[:username],
-            'Authorization'     =>options[:password]
+        @root_id = options[:root_id]
+        rest_params = { base_url: options[:url]}
+        if /^Bearer /.match?(options[:password])
+          rest_params[:headers] = {
+            'X-Aspera-AccessKey' => options[:username],
+            'Authorization'      => options[:password]
           }
-          raise "root_id is required for access key" if @root_id.nil?
+          raise 'root_id is required for access key' if @root_id.nil?
         else
-          rest_params[:auth]={
+          rest_params[:auth] = {
             type:     :basic,
             username: options[:username],
             password: options[:password]
           }
         end
-        @node_api=Rest.new(rest_params)
+        @node_api = Rest.new(rest_params)
         # TODO: currently only supports one transfer. This is bad shortcut. but ok for CLI.
-        @transfer_id=nil
+        @transfer_id = nil
       end
 
       # used internally to ensure node api is set before using.
@@ -44,61 +47,61 @@ module Aspera
 
       # use this to set the node_api end point before using the class.
       def node_api=(new_value)
-        if !@node_api.nil? and !new_value.nil?
+        if !@node_api.nil? && !new_value.nil?
           Log.log.warn('overriding existing node api value')
         end
-        @node_api=new_value
+        @node_api = new_value
       end
 
       # generic method
-      def start_transfer(transfer_spec,options=nil)
+      def start_transfer(transfer_spec,_options=nil)
         # add root id if access key
-        if ! @root_id.nil?
+        if !@root_id.nil?
           case transfer_spec['direction']
-          when 'send';transfer_spec['source_root_id']=@root_id
-          when 'receive';transfer_spec['destination_root_id']=@root_id
+          when Fasp::TransferSpec::DIRECTION_SEND then transfer_spec['source_root_id'] = @root_id
+          when Fasp::TransferSpec::DIRECTION_RECEIVE then transfer_spec['destination_root_id'] = @root_id
           else raise "unexpected direction in ts: #{transfer_spec['direction']}"
           end
         end
         # manage special additional parameter
-        if transfer_spec.has_key?('EX_ssh_key_paths') and transfer_spec['EX_ssh_key_paths'].is_a?(Array) and !transfer_spec['EX_ssh_key_paths'].empty?
+        if transfer_spec.has_key?('EX_ssh_key_paths') && transfer_spec['EX_ssh_key_paths'].is_a?(Array) && !transfer_spec['EX_ssh_key_paths'].empty?
           # not standard, so place standard field
           if transfer_spec.has_key?('ssh_private_key')
             Log.log.warn('Both ssh_private_key and EX_ssh_key_paths are present, using ssh_private_key')
           else
             Log.log.warn('EX_ssh_key_paths has multiple keys, using first one only') unless transfer_spec['EX_ssh_key_paths'].length.eql?(1)
-            transfer_spec['ssh_private_key']=File.read(transfer_spec['EX_ssh_key_paths'].first)
+            transfer_spec['ssh_private_key'] = File.read(transfer_spec['EX_ssh_key_paths'].first)
             transfer_spec.delete('EX_ssh_key_paths')
           end
         end
-        if transfer_spec['tags'].is_a?(Hash) and transfer_spec['tags']['aspera'].is_a?(Hash)
-          transfer_spec['tags']['aspera']['xfer_retry']||=150
+        if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags']['aspera'].is_a?(Hash)
+          transfer_spec['tags']['aspera']['xfer_retry'] ||= 150
         end
         # optimisation in case of sending to the same node
         if transfer_spec['remote_host'].eql?(URI.parse(node_api_.params[:base_url]).host)
-          transfer_spec['remote_host']='localhost'
+          transfer_spec['remote_host'] = 'localhost'
         end
-        resp=node_api_.create('ops/transfers',transfer_spec)[:data]
-        @transfer_id=resp['id']
+        resp = node_api_.create('ops/transfers',transfer_spec)[:data]
+        @transfer_id = resp['id']
         Log.log.debug("tr_id=#{@transfer_id}")
         return @transfer_id
       end
 
       # generic method
       def wait_for_transfers_completion
-        started=false
-        spinner=nil
+        started = false
+        spinner = nil
         # lets emulate management events to display progress bar
         loop do
           # status is empty sometimes with status 200...
-          trdata=node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {"status"=>"unknown"} rescue {"status"=>"waiting(read error)"}
+          trdata = node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {'status' => 'unknown'} rescue {'status' => 'waiting(read error)'}
           case trdata['status']
           when 'completed'
             notify_end(@transfer_id)
             break
           when 'waiting','partially_completed','unknown','waiting(read error)'
             if spinner.nil?
-              spinner = TTY::Spinner.new("[:spinner] :title", format: :classic)
+              spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
               spinner.start
             end
             spinner.update(title: trdata['status'])
@@ -106,18 +109,18 @@ module Aspera
             #puts trdata
           when 'running'
             #puts "running: sessions:#{trdata["sessions"].length}, #{trdata["sessions"].map{|i| i['bytes_transferred']}.join(',')}"
-            if !started and trdata['precalc'].is_a?(Hash) and
+            if !started && trdata['precalc'].is_a?(Hash) &&
             trdata['precalc']['status'].eql?('ready')
               notify_begin(@transfer_id,trdata['precalc']['bytes_expected'])
-              started=true
+              started = true
             else
               notify_progress(@transfer_id,trdata['bytes_transferred'])
             end
           else
             Log.log.warn("trdata -> #{trdata}")
-            raise Fasp::Error.new("#{trdata['status']}: #{trdata['error_desc']}")
+            raise Fasp::Error, "#{trdata['status']}: #{trdata['error_desc']}"
           end
-          sleep 1
+          sleep(1)
         end
         #TODO get status of sessions
         return []

data/lib/aspera/fasp/agent_trsdk.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'aspera/fasp/agent_base'
 require 'aspera/fasp/installation'
 require 'json'
@@ -7,23 +9,18 @@ module Aspera
     class AgentTrsdk < AgentBase
       DEFAULT_OPTIONS = {
         address: '127.0.0.1',
-        port:    55002,
-      }
+        port:    55_002
+      }.freeze
       private_constant :DEFAULT_OPTIONS
 
       # options come from transfer_info
       def initialize(user_opts)
-        raise "expecting Hash (or nil), but have #{user_opts.class}" unless user_opts.nil? or user_opts.is_a?(Hash)
+        raise "expecting Hash (or nil), but have #{user_opts.class}" unless user_opts.nil? || user_opts.is_a?(Hash)
         # set default options and override if specified
-        options=DEFAULT_OPTIONS.clone
-        if !user_opts.nil?
-          user_opts.each do |k,v|
-            if DEFAULT_OPTIONS.has_key?(k)
-              options[k]=v
-            else
-              raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map{|i|i.to_s}.join(",")}"
-            end
-          end
+        options = DEFAULT_OPTIONS.dup
+        user_opts&.each do |k,v|
+          raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.has_key?(k)
+          options[k] = v
         end
         Log.log.debug("options= #{options}")
         super()
@@ -34,39 +31,39 @@ module Aspera
         begin
           get_info_response = @transfer_client.get_info(Transfersdk::InstanceInfoRequest.new)
           Log.log.debug("daemon info: #{get_info_response}")
-        rescue GRPC::Unavailable => e
-          Log.log.warn("no daemon present, starting daemon...")
+        rescue GRPC::Unavailable
+          Log.log.warn('no daemon present, starting daemon...')
           # location of daemon binary
-          bin_folder=File.realpath(File.join(Installation.instance.sdk_ruby_folder,'..'))
+          bin_folder = File.realpath(File.join(Installation.instance.sdk_ruby_folder,'..'))
           # config file and logs are created in same folder
           conf_file = File.join(bin_folder,'sdk.conf')
           log_base = File.join(bin_folder,'transferd')
           # create a config file for daemon
           config = {
-            address: options[:address],
-            port:    options[:port],
+            address:      options[:address],
+            port:         options[:port],
             fasp_runtime: {
-            use_embedded: false,
-            user_defined: {
-            bin: bin_folder,
-            etc: bin_folder,
-            }
+              use_embedded: false,
+              user_defined: {
+                bin: bin_folder,
+                etc: bin_folder
+              }
             }
           }
           File.write(conf_file,config.to_json)
-          trd_pid = Process.spawn(Installation.instance.path(:transferd),'--config' , conf_file, out: "#{log_base}.out", err: "#{log_base}.err")
+          trd_pid = Process.spawn(Installation.instance.path(:transferd),'--config', conf_file, out: "#{log_base}.out", err: "#{log_base}.err")
           Process.detach(trd_pid)
           sleep(2.0)
           retry
         end
       end
 
-      def start_transfer(transfer_spec,options=nil)
+      def start_transfer(transfer_spec,_options=nil)
         # create a transfer request
         transfer_request = Transfersdk::TransferRequest.new(
-        transferType: Transfersdk::TransferType::FILE_REGULAR, # transfer type (file/stream)
-        config: Transfersdk::TransferConfig.new, # transfer configuration
-        transferSpec: transfer_spec.to_json) # transfer definition
+          transferType: Transfersdk::TransferType::FILE_REGULAR, # transfer type (file/stream)
+          config: Transfersdk::TransferConfig.new, # transfer configuration
+          transferSpec: transfer_spec.to_json) # transfer definition
         # send start transfer request to the transfer manager daemon
         start_transfer_response = @transfer_client.start_transfer(transfer_request)
         Log.log.debug("start transfer response #{start_transfer_response}")
@@ -75,22 +72,22 @@ module Aspera
       end
 
       def wait_for_transfers_completion
-        started=false
+        started = false
         # monitor transfer status
         @transfer_client.monitor_transfers(Transfersdk::RegistrationRequest.new(transferId: [@transfer_id])) do |response|
           Log.dump(:response, response.to_h)
           #Log.log.debug("#{response.sessionInfo.preTransferBytes} #{response.transferInfo.bytesTransferred}")
           case response.status
           when :RUNNING
-            if !started and !response.sessionInfo.preTransferBytes.eql?(0)
+            if !started && !response.sessionInfo.preTransferBytes.eql?(0)
               notify_begin(@transfer_id,response.sessionInfo.preTransferBytes)
-              started=true
+              started = true
             elsif started
               notify_progress(@transfer_id,response.transferInfo.bytesTransferred)
             end
           when :FAILED, :COMPLETED, :CANCELED
             notify_end(@transfer_id)
-            raise Fasp::Error.new(JSON.parse(response.message)['Description']) unless :COMPLETED.eql?(response.status)
+            raise Fasp::Error, JSON.parse(response.message)['Description'] unless :COMPLETED.eql?(response.status)
             break
           when :QUEUED,:UNKNOWN_STATUS,:PAUSED,:ORPHANED
             # ignore

data/lib/aspera/fasp/error.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'aspera/fasp/error_info'
 
 module Aspera
@@ -11,7 +13,7 @@ module Aspera
       end
 
       def info
-        r=Fasp::ERROR_INFO[@err_code] || {r: false , c: 'UNKNOWN', m: 'unknown', a: 'unknown'}
+        r = Fasp::ERROR_INFO[@err_code] || {r: false, c: 'UNKNOWN', m: 'unknown', a: 'unknown'}
         return r.merge({i: @err_code})
       end