aspera-cli 4.12.0 → 4.14.0

data/lib/aspera/fasp/agent_httpgw.rb

@@ -9,6 +9,17 @@ require 'websocket'
 require 'base64'
 require 'json'
 
+# HTTP GW Upload protocol
+# -----------------------
+# v1
+# 1 - MessageType: String (Transfer Spec) JSON : type: transfer_spec, acknowledged with "end upload"
+# 2.. - MessageType: String (Slice Upload start) JSON : type: slice_upload, acknowledged with "end upload"
+# v2
+# 1 - MessageType: String (Transfer Spec) JSON : type: transfer_spec, acknowledged with "end upload"
+# 2 - MessageType: String (Slice Upload start) JSON : type: slice_upload, acknowledged with "end_slice_upload"
+# 3.. - MessageType: ByteArray (File Size) Chunks : acknowledged with "end upload"
+# last - MessageType: String (Slice Upload end) JSON : type: slice_upload, acknowledged with "end_slice_upload"
+
 # ref: https://api.ibm.com/explorer/catalog/aspera/product/ibm-aspera/api/http-gateway-api/doc/guides-toc
 # https://developer.ibm.com/apis/catalog?search=%22aspera%20http%22
 module Aspera
@@ -16,35 +27,66 @@ module Aspera
     # start a transfer using Aspera HTTP Gateway, using web socket session for uploads
     class AgentHttpgw < Aspera::Fasp::AgentBase
       # message returned by HTTP GW in case of success
-      MSG_END_UPLOAD = 'end upload'
-      MSG_END_SLICE = 'end_slice_upload'
+      MSG_RECV_DATA_RECEIVED_SIGNAL = 'end upload'
+      MSG_RECV_SLICE_UPLOAD_SIGNAL = 'end_slice_upload'
+      MSG_SEND_SLICE_UPLOAD = 'slice_upload'
+      MSG_SEND_TRANSFER_SPEC = 'transfer_spec'
+      # upload API versions
+      API_V1 = 'v1'
+      API_V2 = 'v2'
       # options available in CLI (transfer_info)
       DEFAULT_OPTIONS = {
         url:                    nil,
         upload_chunk_size:      64_000,
-        upload_bar_refresh_sec: 0.5
+        upload_bar_refresh_sec: 0.5,
+        api_version:            API_V2,
+        synchronous:            true
       }.freeze
       DEFAULT_BASE_PATH = '/aspera/http-gwy'
-      # upload endpoints
-      V1_UPLOAD = '/v1/upload'
-      V2_UPLOAD = '/v2/upload'
-      private_constant :DEFAULT_OPTIONS, :MSG_END_UPLOAD, :MSG_END_SLICE, :V1_UPLOAD, :V2_UPLOAD
+      LOG_WS_MAIN = 'ws: send: '.green
+      LOG_WS_THREAD = 'ws: ack: '.red
+      private_constant :DEFAULT_OPTIONS, :MSG_RECV_DATA_RECEIVED_SIGNAL, :MSG_RECV_SLICE_UPLOAD_SIGNAL, :API_V1, :API_V2
 
       # send message on http gw web socket
-      def ws_snd_json(data)
-        @slice_uploads += 1 if data.key?(:slice_upload)
-        Log.log.debug{JSON.generate(data)}
-        ws_send(JSON.generate(data))
+      def ws_snd_json(msg_type, payload)
+        if msg_type.eql?(MSG_SEND_SLICE_UPLOAD) && @options[:api_version].eql?(API_V2)
+          @shared_info[:count][:sent_v2_slice] += 1
+        else
+          @shared_info[:count][:sent_other] += 1
+        end
+        Log.log.debug do
+          log_data = payload.dup
+          log_data[:data] = "[data #{log_data[:data].length} bytes]" if log_data.key?(:data)
+          "send_txt: #{msg_type}: #{JSON.generate(log_data)}"
+        end
+        ws_send(JSON.generate({msg_type => payload}))
       end
 
-      def ws_send(data, type: :text)
-        frame = ::WebSocket::Frame::Outgoing::Client.new(data: data, type: type, version: @ws_handshake.version)
+      def ws_send(data_to_send, type: :text)
+        Log.log.debug{"#{LOG_WS_MAIN}send low: type: #{type}"}
+        @shared_info[:count][:sent_other] += 1 if type.eql?(:binary)
+        Log.log.debug{"#{LOG_WS_MAIN}counts: #{@shared_info[:count]}"}
+        frame = ::WebSocket::Frame::Outgoing::Client.new(data: data_to_send, type: type, version: @ws_handshake.version)
         @ws_io.write(frame.to_s)
       end
 
+      # wait for all message sent to be acknowledged by HTTPGW server
+      def wait_for_sent_msg_ack_or_exception
+        return unless @options[:synchronous]
+        @shared_info[:mutex].synchronize do
+          while (@shared_info[:count][:received_data] != @shared_info[:count][:sent_other]) ||
+              (@shared_info[:count][:received_v2_slice] != @shared_info[:count][:sent_v2_slice])
+            Log.log.debug{"#{LOG_WS_MAIN}wait: counts: #{@shared_info[:count]}"}
+            @shared_info[:cond_var].wait(@shared_info[:mutex], 1.0)
+            raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
+          end
+        end
+        Log.log.debug{"#{LOG_WS_MAIN}sync ok: counts: #{@shared_info[:count]}"}
+      end
+
       def upload(transfer_spec)
         # total size of all files
-        total_size = 0
+        total_bytes_to_transfer = 0
         # we need to keep track of actual file path because transfer spec is modified to be sent in web socket
         source_paths = []
         # get source root or nil
@@ -61,50 +103,65 @@ module Aspera
           # GW expects a simple file name in 'source' but if user wants to change the name, we take it
           item['source'] = File.basename(item['destination'].nil? ? item['source'] : item['destination'])
           item['file_size'] = File.size(full_src_filepath)
-          total_size += item['file_size']
+          total_bytes_to_transfer += item['file_size']
           # save so that we can actually read the file later
           source_paths.push(full_src_filepath)
         end
         # identify this session uniquely
         session_id = SecureRandom.uuid
-        @slice_uploads = 0
-        # web socket endpoint: by default use v2 (newer gateways), without base64 encoding
-        upload_api_version = V2_UPLOAD
-        # is the latest supported? else revert to old api
-        upload_api_version = V1_UPLOAD unless @api_info['endpoints'].any?{|i|i.include?(upload_api_version)}
-        Log.log.debug{"api version: #{upload_api_version}"}
-        url = File.join(@gw_api.params[:base_url], upload_api_version)
-        # uri = URI.parse(url)
+        upload_url = File.join(@gw_api.params[:base_url], @options[:api_version], 'upload')
+        # uri = URI.parse(upload_url)
         # open web socket to end point (equivalent to Net::HTTP.start)
-        http_socket = Rest.start_http_session(url)
+        http_socket = Rest.start_http_session(upload_url)
+        # little hack to get the socket opened for HTTP, handy because HTTP debug will be available
         @ws_io = http_socket.instance_variable_get(:@socket)
         # @ws_io.debug_output = Log.log
-        @ws_handshake = ::WebSocket::Handshake::Client.new(url: url, headers: {})
+        @ws_handshake = ::WebSocket::Handshake::Client.new(url: upload_url, headers: {})
         @ws_io.write(@ws_handshake.to_s)
         sleep(0.1)
         @ws_handshake << @ws_io.readuntil("\r\n\r\n")
         raise 'Error in websocket handshake' unless @ws_handshake.finished?
-        Log.log.debug('ws: handshake success')
+        Log.log.debug{"#{LOG_WS_MAIN}handshake success"}
         # data shared between main thread and read thread
-        shared_info = {
+        @shared_info = {
           read_exception: nil, # error message if any in callback
-          end_uploads:    0 # number of files totally sent
-          # mutex: Mutex.new
-          # cond_var: ConditionVariable.new
+          count:          {
+            received_data:     0, # number of files received on other side
+            received_v2_slice: 0, # number of slices received on other side
+            sent_other:        0,
+            sent_v2_slice:     0
+          },
+          mutex:          Mutex.new,
+          cond_var:       ConditionVariable.new
         }
         # start read thread
         ws_read_thread = Thread.new do
-          Log.log.debug('ws: thread: started')
-          frame = ::WebSocket::Frame::Incoming::Client.new
+          Log.log.debug{"#{LOG_WS_THREAD}read started"}
+          frame = ::WebSocket::Frame::Incoming::Client.new(version: @ws_handshake.version)
           loop do
             begin # rubocop:disable Style/RedundantBegin
-              frame << @ws_io.readuntil("\n")
+              # unless (recv_data = @ws_io.getc)
+              #  sleep(0.1)
+              #  next
+              # end
+              # frame << recv_data
+              # frame << @ws_io.readuntil("\n")
+              # frame << @ws_io.read_all
+              frame << @ws_io.read(1)
               while (msg = frame.next)
-                Log.log.debug{"ws: thread: message: #{msg.data} #{shared_info[:end_uploads]}"}
+                Log.log.debug{"#{LOG_WS_THREAD}type: #{msg.class}"}
                 message = msg.data
-                if message.eql?(MSG_END_UPLOAD)
-                  shared_info[:end_uploads] += 1
-                elsif message.eql?(MSG_END_SLICE)
+                Log.log.debug{"#{LOG_WS_THREAD}message: [#{message}]"}
+                if message.eql?(MSG_RECV_DATA_RECEIVED_SIGNAL)
+                  @shared_info[:mutex].synchronize do
+                    @shared_info[:count][:received_data] += 1
+                    @shared_info[:cond_var].signal
+                  end
+                elsif message.eql?(MSG_RECV_SLICE_UPLOAD_SIGNAL)
+                  @shared_info[:mutex].synchronize do
+                    @shared_info[:count][:received_v2_slice] += 1
+                    @shared_info[:cond_var].signal
+                  end
                 else
                   message.chomp!
                   error_message =
@@ -117,19 +174,25 @@ module Aspera
                     end
                   raise error_message
                 end
-              end
+                Log.log.debug{"#{LOG_WS_THREAD}counts: #{@shared_info[:count]}"}
+              end # while
             rescue => e
-              shared_info[:read_exception] = e unless e.is_a?(EOFError)
+              Log.log.debug{"#{LOG_WS_THREAD}Exception: #{e}"}
+              @shared_info[:mutex].synchronize do
+                @shared_info[:read_exception] = e unless e.is_a?(EOFError)
+                @shared_info[:cond_var].signal
+              end
               break
-            end
-          end
-          Log.log.debug{"ws: thread: stopping (exc=#{shared_info[:read_exception]},cls=#{shared_info[:read_exception].class})"}
+            end # begin
+          end # loop
+          Log.log.debug{"#{LOG_WS_THREAD}stopping (exc=#{@shared_info[:read_exception]},cls=#{@shared_info[:read_exception].class})"}
         end
         # notify progress bar
-        notify_begin(session_id, total_size)
+        notify_begin(session_id, total_bytes_to_transfer)
         # first step send transfer spec
         Log.dump(:ws_spec, transfer_spec)
-        ws_snd_json(transfer_spec: transfer_spec)
+        ws_snd_json(MSG_SEND_TRANSFER_SPEC, transfer_spec)
+        wait_for_sent_msg_ack_or_exception
         # current file index
         file_index = 0
         # aggregate size sent
@@ -148,7 +211,7 @@ module Aspera
             # current slice index
             slice_index = 0
             until file.eof?
-              data = file.read(@options[:upload_chunk_size])
+              file_bin_data = file.read(@options[:upload_chunk_size])
               slice_data = {
                 name:         file_name,
                 type:         file_mime_type,
@@ -159,22 +222,26 @@ module Aspera
               }
               # Log.dump(:slice_data,slice_data) #if slice_index.eql?(0)
               # interrupt main thread if read thread failed
-              raise shared_info[:read_exception] unless shared_info[:read_exception].nil?
+              raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
               begin
-                if upload_api_version.eql?(V1_UPLOAD)
-                  slice_data[:data] = Base64.strict_encode64(data)
-                  ws_snd_json(slice_upload: slice_data)
+                if @options[:api_version].eql?(API_V1)
+                  slice_data[:data] = Base64.strict_encode64(file_bin_data)
+                  ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_data)
                 else
-                  ws_snd_json(slice_upload: slice_data) if slice_index.eql?(0)
-                  ws_send(data, type: :binary)
-                  Log.log.debug{"ws: sent buffer: #{file_index} / #{slice_index}"}
-                  ws_snd_json(slice_upload: slice_data) if slice_index.eql?(slice_total - 1)
+                  ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_data) if slice_index.eql?(0)
+                  ws_send(file_bin_data, type: :binary)
+                  Log.log.debug{"#{LOG_WS_MAIN}sent bin buffer: #{file_index} / #{slice_index}"}
+                  ws_snd_json(MSG_SEND_SLICE_UPLOAD, slice_data) if slice_index.eql?(slice_total - 1)
                 end
+                wait_for_sent_msg_ack_or_exception
               rescue Errno::EPIPE => e
-                raise shared_info[:read_exception] unless shared_info[:read_exception].nil?
+                raise @shared_info[:read_exception] unless @shared_info[:read_exception].nil?
+                raise e
+              rescue Net::ReadTimeout => e
+                Log.log.warn{'A timeout condition using HTTPGW may signal a permission problem on destination. Check ascp logs on httpgw.'}
                 raise e
               end
-              sent_bytes += data.length
+              sent_bytes += file_bin_data.length
               current_time = Time.now
               if last_progress_time.nil? || ((current_time - last_progress_time) > @options[:upload_bar_refresh_sec])
                 notify_progress(session_id, sent_bytes)
@@ -186,9 +253,9 @@ module Aspera
           file_index += 1
         end
 
-        Log.log.debug('Finished upload')
+        Log.log.debug('Finished upload, waiting for end of read thread.')
         ws_read_thread.join
-        Log.log.debug{"result: #{shared_info[:end_uploads]} / #{@slice_uploads}"}
+        Log.log.debug{"Read thread joined, result: #{@shared_info[:count][:received_data]} / #{@shared_info[:count][:sent_other]}"}
         ws_send(nil, type: :close) unless @ws_io.nil?
         @ws_io = nil
         http_socket&.finish
@@ -258,7 +325,7 @@ module Aspera
       private
 
       def initialize(opts)
-        Log.log.debug{"local options= #{opts}"}
+        Log.dump(:in_options, opts)
         # set default options and override if specified
         @options = DEFAULT_OPTIONS.dup
         raise "httpgw agent parameters (transfer_info): expecting Hash, but have #{opts.class}" unless opts.is_a?(Hash)
@@ -266,13 +333,24 @@ module Aspera
           raise "httpgw agent parameter: Unknown: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.key?(k)
           @options[k] = v
         end
-        raise 'missing param: url' if @options[:url].nil?
+        if @options[:url].nil?
+          available = DEFAULT_OPTIONS.map { |k, v| "#{k}(#{v})"}.join(', ')
+          raise "Missing mandatory parameter for HTTP GW in transfer_info: url. Allowed parameters: #{available}."
+        end
         # remove /v1 from end
         @options[:url].gsub(%r{/v1/*$}, '')
         super()
         @gw_api = Rest.new({base_url: @options[:url]})
         @api_info = @gw_api.read('v1/info')[:data]
-        Log.log.info(@api_info.to_s)
+        Log.dump(:api_info, @api_info)
+        if @options[:api_version].nil?
+          # web socket endpoint: by default use v2 (newer gateways), without base64 encoding
+          @options[:api_version] = API_V2
+          # is the latest supported? else revert to old api
+          @options[:api_version] = API_V1 unless @api_info['endpoints'].any?{|i|i.include?(@options[:api_version])}
+        end
+        @options.freeze
+        Log.dump(:final_options, @options)
       end
     end # AgentHttpgw
   end

data/lib/aspera/fasp/agent_node.rb

@@ -76,10 +76,13 @@ module Aspera
             transfer_spec.delete('EX_ssh_key_paths')
           end
         end
-        if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags']['aspera'].is_a?(Hash)
-          transfer_spec['tags']['aspera']['xfer_retry'] ||= 150
+        # add mandatory retry parameter for node api
+        ts_tags = transfer_spec['tags']
+        if ts_tags.is_a?(Hash) && ts_tags[Fasp::TransferSpec::TAG_RESERVED].is_a?(Hash)
+          ts_tags[Fasp::TransferSpec::TAG_RESERVED]['xfer_retry'] ||= 150
         end
-        # Optimization in case of sending to the same node (TODO: probably remove this, as /etc/hosts shall be used for that)
+        # Optimization in case of sending to the same node
+        # TODO: probably remove this, as /etc/hosts shall be used for that
         if !transfer_spec['wss_enabled'] && transfer_spec['remote_host'].eql?(URI.parse(node_api_.params[:base_url]).host)
           transfer_spec['remote_host'] = '127.0.0.1'
         end
@@ -116,9 +119,16 @@ module Aspera
             else
               notify_progress(@transfer_id, transfer_data['bytes_transferred'])
             end
+          when 'failed'
+            # Bug in HSTS ? transfer is marked failed, but there is no reason
+            if transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
+              notify_end(@transfer_id)
+              break
+            end
+            raise Fasp::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
           else
             Log.log.warn{"transfer_data -> #{transfer_data}"}
-            raise Fasp::Error, "#{transfer_data['status']}: #{transfer_data['error_desc']}"
+            raise Fasp::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
           end
           sleep(1)
         end

data/lib/aspera/fasp/agent_trsdk.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+# cspell:words Transfersdk
+
 require 'aspera/fasp/agent_base'
 require 'aspera/fasp/installation'
 require 'json'

data/lib/aspera/fasp/error_info.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+# cspell:words mnemo PROTO RCVR NOLIC PMTU BRTT VLINK BWMEAS SSEAR FIPS
+
 module Aspera
   module Fasp
     # from https://www.google.com/search?q=FASP+error+codes

data/lib/aspera/fasp/installation.rb

@@ -62,6 +62,7 @@ module Aspera
 
       # location of SDK files
       def sdk_folder=(v)
+        Log.log.debug{"sdk_folder=#{v}"}
         @sdk_dir = v
         sdk_folder
       end
@@ -125,7 +126,11 @@ module Aspera
       end
 
       # all ascp files (in SDK)
-      FILES = %i[ascp ascp4 ssh_bypass_key_dsa ssh_bypass_key_rsa aspera_license aspera_conf fallback_cert fallback_key].freeze
+      FILES = %i[ascp ascp4 ssh_bypass_dsa_privkey ssh_bypass_rsa_privkey aspera_license aspera_conf fallback_certificate fallback_cert_privkey].freeze
+
+      def check_or_create_sdk_file(filename, force: false, &block)
+        return Environment.write_file_restricted(File.join(sdk_folder, filename), force: force, mode: 0o644, &block)
+      end
 
       # get path of one resource file of currently activated product
       # keys and certs are generated locally... (they are well known values, arch. independent)
@@ -138,23 +143,18 @@ module Aspera
           file = file.gsub('ascp', 'ascp4') if k.eql?(:ascp4)
         when :transferd
           file = transferd_filepath
-        when :ssh_bypass_key_dsa
-          file = Environment.write_file_restricted(File.join(sdk_folder, 'aspera_bypass_dsa.pem')) {get_key('dsa', 1)}
-        when :ssh_bypass_key_rsa
-          file = Environment.write_file_restricted(File.join(sdk_folder, 'aspera_bypass_rsa.pem')) {get_key('rsa', 2)}
+        when :ssh_bypass_dsa_privkey
+          file = check_or_create_sdk_file('aspera_bypass_dsa.pem') {get_key('dsa', 1)}
+        when :ssh_bypass_rsa_privkey
+          file = check_or_create_sdk_file('aspera_bypass_rsa.pem') {get_key('rsa', 2)}
         when :aspera_license
-          file = Environment.write_file_restricted(File.join(sdk_folder, 'aspera-license')) do
-            clear = [
-              Zlib::Inflate.inflate(DataRepository.instance.data(6)),
-              "==SIGNATURE==\n",
-              Base64.strict_encode64(DataRepository.instance.data(7))
-            ]
-            Base64.strict_encode64(clear.join)
+          file = check_or_create_sdk_file('aspera-license') do
+            Zlib::Inflate.inflate(DataRepository.instance.data(6))
           end
         when :aspera_conf
-          file = Environment.write_file_restricted(File.join(sdk_folder, 'aspera.conf')) {DEFAULT_ASPERA_CONF}
-        when :fallback_cert, :fallback_key
-          file_key = File.join(sdk_folder, 'aspera_fallback_key.pem')
+          file = check_or_create_sdk_file('aspera.conf') {DEFAULT_ASPERA_CONF}
+        when :fallback_certificate, :fallback_cert_privkey
+          file_key = File.join(sdk_folder, 'aspera_fallback_cert_private_key.pem')
           file_cert = File.join(sdk_folder, 'aspera_fallback_cert.pem')
           if !File.exist?(file_key) || !File.exist?(file_cert)
             require 'openssl'
@@ -168,10 +168,10 @@ module Aspera
             cert.serial = 0x0
             cert.version = 2
             cert.sign(private_key, OpenSSL::Digest.new('SHA1'))
-            Environment.write_file_restricted(file_key, force: true) {private_key.to_pem}
-            Environment.write_file_restricted(file_cert, force: true) {cert.to_pem}
+            check_or_create_sdk_file('aspera_fallback_cert_private_key.pem', force: true) {private_key.to_pem}
+            check_or_create_sdk_file('aspera_fallback_cert.pem', force: true) {cert.to_pem}
           end
-          file = k.eql?(:fallback_cert) ? file_cert : file_key
+          file = k.eql?(:fallback_certificate) ? file_cert : file_key
         else
           raise "INTERNAL ERROR: #{k}"
         end
@@ -205,7 +205,7 @@ module Aspera
       end
 
       def bypass_keys
-        return %i[ssh_bypass_key_dsa ssh_bypass_key_rsa].map{|i|Installation.instance.path(i)}
+        return %i[ssh_bypass_dsa_privkey ssh_bypass_rsa_privkey].map{|i|Installation.instance.path(i)}
       end
 
       # use in plugin `config`

data/lib/aspera/fasp/parameters.rb

@@ -3,6 +3,7 @@
 require 'aspera/log'
 require 'aspera/command_line_builder'
 require 'aspera/temp_file_manager'
+require 'aspera/fasp/error'
 require 'securerandom'
 require 'base64'
 require 'json'
@@ -19,6 +20,7 @@ module Aspera
       # Short names of columns in manual
       SUPPORTED_AGENTS_SHORT = SUPPORTED_AGENTS.map{|a|a.to_s[0].to_sym}
       FILE_LIST_OPTIONS = ['--file-list', '--file-pair-list'].freeze
+      SUPPORTED_OPTIONS = %i[ascp_args wss].freeze
 
       private_constant :SUPPORTED_AGENTS, :FILE_LIST_OPTIONS
 
@@ -38,8 +40,9 @@ module Aspera
           return @param_description_cache
         end
 
+        # @param to_text [bool] replace HTML entities with text equivalent
         # @return a table suitable to display in manual
-        def man_table(to_text: true)
+        def man_table
           result = []
           description.each do |name, options|
             param = {name: name, type: [options[:accepted_types]].flatten.join(','), description: options[:desc]}
@@ -70,10 +73,17 @@ module Aspera
             if options.key?(:enum)
               param[:description] += "\nAllowed values: #{options[:enum].join(', ')}"
             end
-            param[:description] = param[:description].gsub('/', '/') if to_text
+            # replace "solidus" HTML entity with its text value
+            param[:description] = param[:description].gsub('/', '\\')
             result.push(param)
           end
-          return result
+          return result.sort do |a, b|
+            if a[:name].start_with?('EX_').eql?(b[:name].start_with?('EX_'))
+              a[:name] <=> b[:name]
+            else
+              b[:name] <=> a[:name]
+            end
+          end
         end
 
         # special encoding methods used in YAML (key: :convert)
@@ -95,10 +105,6 @@ module Aspera
             ts.key?('EX_file_pair_list')
         end
 
-        def ts_to_env_args(transfer_spec, wss:, ascp_args:)
-          return Parameters.new(transfer_spec, wss: wss, ascp_args: ascp_args).ascp_args
-        end
-
         # temp file list files are created here
         def file_list_folder=(v)
           @file_list_folder = v
@@ -112,19 +118,20 @@ module Aspera
       end # self
 
       # @param options [Hash] key: :wss: bool, :ascp_args: array of strings
-      def initialize(job_spec, wss:, ascp_args:)
+      def initialize(job_spec, options)
         @job_spec = job_spec
-        @opt_wss = wss
-        @opt_args = ascp_args
-        Log.log.debug{"agent options: #{@opt_wss} #{@opt_args}"}
-        raise 'ascp args must be an Array' unless @opt_args.is_a?(Array)
-        raise 'ascp args must be an Array of String' if @opt_args.any?{|i|!i.is_a?(String)}
+        # check necessary options
+        raise 'Internal: missing options' unless (SUPPORTED_OPTIONS - options.keys).empty?
+        @options = SUPPORTED_OPTIONS.each_with_object({}){|o, h| h[o] = options[o]}
+        Log.dump(:options, @options)
+        raise 'ascp args must be an Array' unless @options[:ascp_args].is_a?(Array)
+        raise 'ascp args must be an Array of String' if @options[:ascp_args].any?{|i|!i.is_a?(String)}
         @builder = Aspera::CommandLineBuilder.new(@job_spec, self.class.description)
       end
 
       def process_file_list
         # is the file list provided through EX_ parameters?
-        ascp_file_list_provided = self.class.ts_has_ascp_file_list(@job_spec, @opt_args)
+        ascp_file_list_provided = self.class.ts_has_ascp_file_list(@job_spec, @options[:ascp_args])
         # set if paths is mandatory in ts
         @builder.params_definition['paths'][:mandatory] = !@job_spec.key?('keepalive') && !ascp_file_list_provided
         # get paths in transfer spec (after setting if it is mandatory)
@@ -146,12 +153,14 @@ module Aspera
               Log.log.debug('placing source file list on command line (no file list file)')
               @builder.add_command_line_options(ts_paths_array.map{|i|i['source']})
             else
+              raise "All elements of paths must have a 'source' key" unless ts_paths_array.all?{|i|i.key?('source')}
+              is_pair_list = ts_paths_array.any?{|i|i.key?('destination')}
+              raise "All elements of paths must be consistent with 'destination' key" if is_pair_list && !ts_paths_array.all?{|i|i.key?('destination')}
               # safer option: generate a file list file if there is storage defined for it
-              # if there is destination in paths, then use file-pair-list
-              # TODO: well, we test only the first one, but anyway it shall be consistent
-              if ts_paths_array.first.key?('destination')
+              # if there is one destination in paths, then use file-pair-list
+              if is_pair_list
                 option = '--file-pair-list'
-                lines = ts_paths_array.each_with_object([]){|e, m|m.push(e['source'], e['destination']); }
+                lines = ts_paths_array.each_with_object([]){|e, m|m.push(e['source'], e['destination'] || e['source']); }
               else
                 option = '--file-list'
                 lines = ts_paths_array.map{|i|i['source']}
@@ -186,7 +195,7 @@ module Aspera
         @job_spec.delete('source_root') if @job_spec.key?('source_root') && @job_spec['source_root'].empty?
 
         # use web socket session initiation ?
-        if @builder.read_param('wss_enabled') && (@opt_wss || !@job_spec.key?('fasp_port'))
+        if @builder.read_param('wss_enabled') && (@options[:wss] || !@job_spec.key?('fasp_port'))
           # by default use web socket session if available, unless removed by user
           @builder.add_command_line_options(['--ws-connect'])
           # TODO: option to give order ssh,ws (legacy http is implied bu ssh)
@@ -218,7 +227,7 @@ module Aspera
         process_file_list
         # optional args, at the end to override previous ones (to allow override)
         @builder.add_command_line_options(@builder.read_param('EX_ascp_args'))
-        @builder.add_command_line_options(@opt_args)
+        @builder.add_command_line_options(@options[:ascp_args])
         # process destination folder
         destination_folder = @builder.read_param('destination_root') || '/'
         # ascp4 does not support base64 encoding of destination

data/lib/aspera/fasp/parameters.yaml

@@ -8,6 +8,7 @@
 # cli.switch     : ascp: switch for ascp command line
 # cli.convert    : ascp: transform value: either a Hash with conversion values, or name of class
 # cli.variable   : ascp: name of env var
+# cspell:words dgram
 ---
 cipher:
   :desc: "In transit encryption type."
@@ -559,7 +560,8 @@ remove_empty_source_directory:
   :cli:
     :type: :opt_without_arg
 EX_at_rest_password:
-  :desc: "DEPRECATED: Prefer to use standard parameter: content_protection_password"
+  :desc: Content protection password
+  :deprecation: "Use standard spec parameter: content_protection_password"
   :agents:
   - :direct
   :cli:
@@ -568,7 +570,7 @@ EX_at_rest_password:
 EX_proxy_password:
   :desc: |-
     Password used for Aspera proxy server authentication.
-    May be overridden by password in URL EX_fasp_proxy_url.
+    May be overridden by password in URL provided in parameter: proxy.
 
 
   :agents:
@@ -703,6 +705,7 @@ EX_file_pair_list:
     :switch: "--file-pair-list"
 EX_ascp_args:
   :desc: Add native command line arguments to ascp
+  :deprecation: Use parameter ascp_args in option transfer_info
   :accepted_types: :array
   :agents:
   - :direct

data/lib/aspera/fasp/resume_policy.rb

@@ -49,11 +49,11 @@ module Aspera
             # failure in ascp
             if e.retryable?
               # exit if we exceed the max number of retry
-              raise Fasp::Error, 'Maximum number of retry reached' if remaining_resumes <= 0
+              raise Fasp::Error, "Maximum number of retry reached (#{@parameters[:iter_max]})" if remaining_resumes <= 0
             else
               # give one chance only to non retryable errors
               unless remaining_resumes.eql?(@parameters[:iter_max])
-                Log.log.error('non-retryable error')
+                Log.log.error('non-retryable error'.red.blink)
                 raise e
               end
             end
@@ -61,7 +61,7 @@ module Aspera
 
           # take this retry in account
           remaining_resumes -= 1
-          Log.log.warn{"resuming in  #{sleep_seconds} seconds (retry left:#{remaining_resumes})"}
+          Log.log.warn{"Resuming in #{sleep_seconds} seconds (retry left:#{remaining_resumes})"}
 
           # wait a bit before retrying, maybe network condition will be better
           sleep(sleep_seconds)

data/lib/aspera/fasp/transfer_spec.rb

@@ -15,11 +15,14 @@ module Aspera
         'ssh_port'    => SSH_PORT,
         'fasp_port'   => UDP_PORT
       }.freeze
-      # define constants for enums of parameters: <paramater>_<enum>, e.g. CIPHER_AES_128
-      Aspera::Fasp::Parameters.description.each do |k, v|
-        next unless v[:enum].is_a?(Array)
-        v[:enum].each do |enum|
-          TransferSpec.const_set("#{k.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
+      # reserved tag for Aspera
+      TAG_RESERVED = 'aspera'
+      # define constants for enums of parameters: <parameter>_<enum>, e.g. CIPHER_AES_128, DIRECTION_SEND, ...
+      Aspera::Fasp::Parameters.description.each do |name, description|
+        next unless description[:enum].is_a?(Array)
+        TransferSpec.const_set("#{name.to_s.upcase}_ENUM_VALUES", description[:enum])
+        description[:enum].each do |enum|
+          TransferSpec.const_set("#{name.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
         end
       end
       class << self