aspera-cli 4.15.0 → 4.17.0

data/lib/aspera/{fasp/agent_direct.rb → agent/direct.rb}

@@ -1,58 +1,61 @@
 # frozen_string_literal: true
 
-require 'aspera/fasp/agent_base'
-require 'aspera/fasp/error'
-require 'aspera/fasp/parameters'
-require 'aspera/fasp/installation'
-require 'aspera/fasp/resume_policy'
-require 'aspera/fasp/transfer_spec'
-require 'aspera/fasp/management'
+require 'aspera/agent/base'
+require 'aspera/ascp/installation'
+require 'aspera/ascp/management'
+require 'aspera/transfer/parameters'
+require 'aspera/transfer/error'
+require 'aspera/transfer/spec'
+require 'aspera/resumer'
 require 'aspera/log'
+require 'aspera/assert'
 require 'socket'
-require 'timeout'
 require 'securerandom'
 require 'shellwords'
 require 'English'
 
 module Aspera
-  module Fasp
+  module Agent
     # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
-    class AgentDirect < Aspera::Fasp::AgentBase
+    class Direct < Base
       # options for initialize (same as values in option transfer_info)
       DEFAULT_OPTIONS = {
-        spawn_timeout_sec: 3,
-        spawn_delay_sec:   2,
         wss:               true, # true: if both SSH and wss in ts: prefer wss
+        ascp_args:         [],
+        spawn_timeout_sec: 2,
+        spawn_delay_sec:   2, # optional delay to start between sessions
         multi_incr_udp:    true,
+        trusted_certs:     [], # list of files with trusted certificates (stores)
         resume:            {},
-        ascp_args:         [],
-        check_ignore:      nil, # callback with host,port
         quiet:             true, # by default no native ascp progress bar
-        trusted_certs:     [] # list of files with trusted certificates (stores)
+        check_ignore_cb:   nil, # callback with host,port
+        management_cb:     nil # callback for management events
       }.freeze
+      LISTEN_LOCAL_ADDRESS = '127.0.0.1'
+      ANY_AVAILABLE_PORT = 0 # 0 means any available port
       # spellchecker: enable
-      private_constant :DEFAULT_OPTIONS
+      private_constant :DEFAULT_OPTIONS, :LISTEN_LOCAL_ADDRESS, :ANY_AVAILABLE_PORT
 
-      # start ascp transfer (non blocking), single or multi-session
-      # job information added to @jobs
+      # method of Base
+      # start ascp transfer(s) (non blocking), single or multi-session
+      # session information added to @sessions
       # @param transfer_spec [Hash] aspera transfer specification
+      # @param token_regenerator [Object] object with method refreshed_transfer_token
       def start_transfer(transfer_spec, token_regenerator: nil)
-        the_job_id = SecureRandom.uuid
         # clone transfer spec because we modify it (first level keys)
         transfer_spec = transfer_spec.clone
-        # if there is aspera tags
-        if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags'][Fasp::TransferSpec::TAG_RESERVED].is_a?(Hash)
+        # if there are aspera tags
+        if transfer_spec.dig('tags', Transfer::Spec::TAG_RESERVED).is_a?(Hash)
           # TODO: what is this for ? only on local ascp ?
           # NOTE: important: transfer id must be unique: generate random id
           # using a non unique id results in discard of tags in AoC, and a package is never finalized
           # all sessions in a multi-session transfer must have the same xfer_id (see admin manual)
-          transfer_spec['tags'][Fasp::TransferSpec::TAG_RESERVED]['xfer_id'] ||= SecureRandom.uuid
+          transfer_spec['tags'][Transfer::Spec::TAG_RESERVED]['xfer_id'] ||= SecureRandom.uuid
           Log.log.debug{"xfer id=#{transfer_spec['xfer_id']}"}
-          # TODO: useful ? node only ?
-          transfer_spec['tags'][Fasp::TransferSpec::TAG_RESERVED]['xfer_retry'] ||= 3600
+          # TODO: useful ? node only ? seems to be a timeout for retry in node
+          transfer_spec['tags'][Transfer::Spec::TAG_RESERVED]['xfer_retry'] ||= 3600
         end
         Log.log.debug{Log.dump('ts', transfer_spec)}
-
         # Compute this before using transfer spec because it potentially modifies the transfer spec
         # (even if the var is not used in single session)
         multi_session_info = nil
@@ -70,37 +73,31 @@ module Aspera
             multi_session_info = nil
           elsif @options[:multi_incr_udp] # multi_session_info[:count] > 0
             # if option not true: keep default udp port for all sessions
-            multi_session_info[:udp_base] = transfer_spec.key?('fasp_port') ? transfer_spec['fasp_port'] : TransferSpec::UDP_PORT
+            multi_session_info[:udp_base] = transfer_spec.key?('fasp_port') ? transfer_spec['fasp_port'] : Transfer::Spec::UDP_PORT
             # delete from original transfer spec, as we will increment values
             transfer_spec.delete('fasp_port')
             # override if specified, else use default value
           end
         end
 
-        # compute known arguments and environment variables
-        env_args = Parameters.new(transfer_spec, @options).ascp_args
-
-        # transfer job can be multi session
-        xfer_job = {
-          id:       the_job_id,
-          sessions: [] # all sessions as below
-        }
-
         # generic session information
         session = {
+          id:                nil, # SessionId from INIT message in mgt port
+          job_id:            SecureRandom.uuid, # job id (regroup sessions)
+          ts:                transfer_spec,     # transfer spec
           thread:            nil,               # Thread object monitoring management port, not nil when pushed to :sessions
           error:             nil,               # exception if failed
           io:                nil,               # management port server socket
-          id:                nil,               # SessionId from INIT message in mgt port
           token_regenerator: token_regenerator, # regenerate bearer token with oauth
-          env_args:          env_args           # env vars and args to ascp (from transfer spec)
+          # env vars and args to ascp (from transfer spec)
+          env_args:          Transfer::Parameters.new(transfer_spec, @options).ascp_args
         }
 
         if multi_session_info.nil?
           Log.log.debug('Starting single session thread')
           # single session for transfer : simple
           session[:thread] = Thread.new(session) {|s|transfer_thread_entry(s)}
-          xfer_job[:sessions].push(session)
+          @sessions.push(session)
         else
           Log.log.debug('Starting multi session threads')
           1.upto(multi_session_info[:count]) do |i|
@@ -108,22 +105,19 @@ module Aspera
             sleep(@options[:spawn_delay_sec]) unless i.eql?(1)
             # do deep copy (each thread has its own copy because it is modified here below and in thread)
             this_session = session.clone
+            this_session[:ts] = this_session[:ts].clone
             this_session[:env_args] = this_session[:env_args].clone
             this_session[:env_args][:args] = this_session[:env_args][:args].clone
+            # set multi session part
             this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_info[:count]}")
             # option: increment (default as per ascp manual) or not (cluster on other side ?)
             this_session[:env_args][:args].unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @options[:multi_incr_udp]
+            # finally start the thread
             this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
-            xfer_job[:sessions].push(this_session)
+            @sessions.push(this_session)
           end
         end
-        Log.log.debug('started session thread(s)')
-
-        # add job to list of jobs
-        @jobs[the_job_id] = xfer_job
-        Log.log.debug{"jobs: #{@jobs.keys.count}"}
-
-        return the_job_id
+        return session[:job_id]
       end # start_transfer
 
       # wait for completion of all jobs started
@@ -132,16 +126,14 @@ module Aspera
         Log.log.debug('wait_for_transfers_completion')
         # set to non-nil to exit loop
         result = []
-        @jobs.each do |_id, job|
-          job[:sessions].each do |session|
-            Log.log.debug{"join #{session[:thread]}"}
-            session[:thread].join
-            result.push(session[:error] || :success)
-          end
+        @sessions.each do |session|
+          Log.log.debug{"join #{session[:thread]}"}
+          session[:thread].join
+          result.push(session[:error] || :success)
         end
         Log.log.debug('all transfers joined')
         # since all are finished and we return the result, clear statuses
-        @jobs.clear
+        @sessions.clear
         return result
       end
 
@@ -150,12 +142,6 @@ module Aspera
         Log.log.debug('fasp local shutdown')
       end
 
-      # cspell:disable
-      # begin 'Type' => 'NOTIFICATION', 'PreTransferBytes' => size
-      # progress 'Type' => 'STATS', 'Bytescont' => size
-      # end 'Type' => 'DONE'
-      # cspell:enable
-
       # @param event management port event
       def process_progress(event)
         session_id = event['SessionId']
@@ -198,63 +184,66 @@ module Aspera
       # start ascp with management port.
       # raises FaspError on error
       # if there is a thread info: set and broadcast session id
+      # runs in separate thread
       # @param env_args a hash containing :args :env :ascp_version
       # @param session this session information
       # could be private method
       def start_transfer_with_args_env(env_args, session)
-        raise 'env_args must be Hash' unless env_args.is_a?(Hash)
-        raise 'session must be Hash' unless session.is_a?(Hash)
+        Aspera.assert_type(env_args, Hash)
+        Aspera.assert_type(session, Hash)
+        Log.log.debug{"env_args=#{env_args.inspect}"}
+        notify_progress(session_id: nil, type: :pre_start, info: 'starting')
         begin
-          Log.log.debug{"env_args=#{env_args.inspect}"}
+          ascp_pid = nil
+          # we use Socket directly, instead of TCPServer, as it gives access to lower level options
+          socket_class = RUBY_ENGINE.eql?('jruby') ? ServerSocket : Socket
+          mgt_server_socket = socket_class.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
+          # open any available (0) local TCP port for use as ascp management port
+          mgt_server_socket.bind(Addrinfo.tcp(LISTEN_LOCAL_ADDRESS, ANY_AVAILABLE_PORT))
+          # build arguments and add mgt port
+          ascp_arguments = ['-M', mgt_server_socket.local_address.ip_port.to_s].concat(env_args[:args])
           # get location of ascp executable
-          ascp_path = @mutex.synchronize do
-            Fasp::Installation.instance.path(env_args[:ascp_version])
-          end
-          # (optional) check it exists
-          raise Fasp::Error, "no such file: #{ascp_path}" unless File.exist?(ascp_path)
-          notify_progress(session_id: nil, type: :pre_start, info: 'starting ascp')
-          # open an available (0) local TCP port as ascp management
-          mgt_sock = TCPServer.new('127.0.0.1', 0)
-          # clone arguments as we eed to modify with mgt port
-          ascp_arguments = env_args[:args].clone
-          # add management port on the selected local port
-          ascp_arguments.unshift('-M', mgt_sock.addr[1].to_s)
+          ascp_path = Ascp::Installation.instance.path(env_args[:ascp_version])
           # display ascp command line
           Log.log.debug do
             [
-              'execute:',
+              'execute:'.red,
               env_args[:env].map{|k, v| "#{k}=#{Shellwords.shellescape(v)}"},
               Shellwords.shellescape(ascp_path),
               ascp_arguments.map{|a|Shellwords.shellescape(a)}
             ].flatten.join(' ')
           end
           # start ascp in separate process
-          ascp_pid = Process.spawn(env_args[:env], [ascp_path, ascp_path], *ascp_arguments)
-          # in parent, wait for connection to socket max 3 seconds
-          Log.log.debug{"before accept for pid (#{ascp_pid})"}
-          # init management socket
-          ascp_mgt_io = nil
+          ascp_pid = Process.spawn(env_args[:env], [ascp_path, ascp_path], *ascp_arguments, close_others: true)
+          Log.log.debug{"spawned ascp pid #{ascp_pid}"}
           notify_progress(session_id: nil, type: :pre_start, info: 'waiting for ascp')
-          Timeout.timeout(@options[:spawn_timeout_sec]) do
-            ascp_mgt_io = mgt_sock.accept
-            # management messages include file names which may be utf8
-            # by default socket is US-ASCII
-            # TODO: use same value as Encoding.default_external
-            ascp_mgt_io.set_encoding(Encoding::UTF_8)
-          end
-          Log.log.debug{"after accept (#{ascp_mgt_io})"}
+          mgt_server_socket.listen(1)
+          # TODO: timeout does not work when Process.spawn is used... until process exits, then it works
+          Log.log.debug{"before select, timeout: #{@options[:spawn_timeout_sec]}"}
+          readable, _, _ = IO.select([mgt_server_socket], nil, nil, @options[:spawn_timeout_sec])
+          Log.log.debug('after select, before accept')
+          Aspera.assert(readable, exception_class: Transfer::Error){'timeout waiting mgt port connect (select not readable)'}
+          # There is a connection to accept
+          client_socket, _client_addrinfo = mgt_server_socket.accept
+          Log.log.debug('after accept')
+          ascp_mgt_io = client_socket.to_io
+          # management messages include file names which may be utf8
+          # by default socket is US-ASCII
+          # TODO: use same value as Encoding.default_external
+          ascp_mgt_io.set_encoding(Encoding::UTF_8)
           session[:io] = ascp_mgt_io
-          processor = Management.new
+          processor = Ascp::Management.new
           # read management port, until socket is closed (gets returns nil)
           while (line = ascp_mgt_io.gets)
             event = processor.process_line(line.chomp)
             next unless event
             # event is ready
-            Log.log.debug{Log.dump(:management_port, event)}
-            # Log.log.trace1{"event: #{JSON.generate(Management.enhanced_event_format(event))}"}
+            Log.log.trace1{Log.dump(:management_port, event)}
+            @options[:management_cb]&.call(event)
             process_progress(event)
             Log.log.error((event['Description']).to_s) if event['Type'].eql?('FILEERROR') # cspell:disable-line
           end
+          Log.log.debug('management io closed')
           last_event = processor.last_event
           # check that last status was received before process exit
           if last_event.is_a?(Hash)
@@ -264,10 +253,10 @@ module Aspera
                   session[:token_regenerator].respond_to?(:refreshed_transfer_token)
                 # regenerate token here, expired, or error on it
                 # Note: in multi-session, each session will have a different one.
-                Log.log.warn('Regenerating bearer token')
+                Log.log.warn('Regenerating token for transfer')
                 env_args[:env]['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
               end
-              raise Fasp::Error.new(last_event['Description'], last_event['Code'].to_i)
+              raise Transfer::Error.new(last_event['Description'], last_event['Code'].to_i)
             when 'DONE'
               nil
             else
@@ -275,42 +264,52 @@ module Aspera
             end # case
           end
         rescue SystemCallError => e
-          # Process.spawn
-          raise Fasp::Error, e.message
-        rescue Timeout::Error
-          raise Fasp::Error, 'timeout waiting mgt port connect'
+          # Process.spawn failed, or socket error
+          raise Transfer::Error, e.message
         rescue Interrupt
-          raise Fasp::Error, 'transfer interrupted by user'
+          raise Transfer::Error, 'transfer interrupted by user'
         ensure
-          # if ascp was successfully started
+          mgt_server_socket.close
+          # if ascp was successfully started, check its status
           unless ascp_pid.nil?
             # "wait" for process to avoid zombie
             Process.wait(ascp_pid)
             status = $CHILD_STATUS
             ascp_pid = nil
             session.delete(:io)
-            if !status.success?
-              message = "ascp failed with code #{status.exitstatus}"
-              # raise error only if there was not already an exception
-              raise Fasp::Error, message unless $ERROR_INFO
-              # else just debug, as main exception is already here
-              Log.log.debug(message)
+            # status is nil if an exception occurred before starting ascp
+            if !status&.success?
+              message = status.nil? ? 'ascp not started' : "ascp failed (#{status})"
+              # raise error only if there was not already an exception (ERROR_INFO)
+              raise Transfer::Error, message unless $ERROR_INFO
+              # else display this message also, as main exception is already here
+              Log.log.error(message)
             end
           end
         end # begin-ensure
       end # start_transfer_with_args_env
 
-      # send command of management port to ascp session
+      # @return [Array] list of sessions for a job
+      def sessions_by_job(job_id)
+        @sessions.select{|s| s[:job_id].eql?(job_id)}
+      end
+
+      # @return [Hash] session information
+      def session_by_id(id)
+        matches = @sessions.select{|s| s[:id].eql?(id)}
+        raise 'no such session' if matches.empty?
+        raise 'more than one session' if matches.length > 1
+        return matches.first
+      end
+
+      # send command of management port to ascp session (used in `asession)
       # @param job_id identified transfer process
       # @param session_index index of session (for multi session)
       # @param data command on mgt port, examples:
       # {'type'=>'START','source'=>_path_,'destination'=>_path_}
       # {'type'=>'DONE'}
-      def send_command(job_id, session_index, data)
-        job = @jobs[job_id]
-        raise 'no such job' if job.nil?
-        session = job[:sessions][session_index]
-        raise 'no such session' if session.nil?
+      def send_command(job_id, data)
+        session = session_by_id(job_id)
         Log.log.debug{"command: #{data}"}
         # build command
         command = data
@@ -321,20 +320,21 @@ module Aspera
           .join("\n")
         session[:io].puts(command)
       end
+      attr_reader :sessions
 
       private
 
       # @param options : keys(symbol): see DEFAULT_OPTIONS
       def initialize(options={})
         super(options)
+        # set default options and override if specified
+        @options = Base.options(default: DEFAULT_OPTIONS, options: options)
+        Log.log.debug{Log.dump(:agent_options, @options)}
+        @resume_policy = Resumer.new(@options[:resume].symbolize_keys)
         # all transfer jobs, key = SecureRandom.uuid, protected by mutex, cond var on change
-        @jobs = {}
+        @sessions = []
         # mutex protects global data accessed by threads
         @mutex = Mutex.new
-        # set default options and override if specified
-        @options = AgentBase.options(default: DEFAULT_OPTIONS, options: options)
-        @resume_policy = ResumePolicy.new(@options[:resume].symbolize_keys)
-        Log.log.debug{Log.dump(:agent_options, @options)}
       end
 
       # transfer thread entry

data/lib/aspera/{fasp/agent_httpgw.rb → agent/httpgw.rb}

@@ -1,9 +1,10 @@
 # frozen_string_literal: true
 
-require 'aspera/fasp/agent_base'
-require 'aspera/fasp/transfer_spec'
-require 'aspera/fasp/faux_file'
+require 'aspera/agent/base'
+require 'aspera/transfer/spec'
+require 'aspera/transfer/faux_file'
 require 'aspera/log'
+require 'aspera/assert'
 require 'aspera/rest'
 require 'securerandom'
 require 'websocket'
@@ -11,7 +12,7 @@ require 'base64'
 require 'json'
 
 module Aspera
-  module Fasp
+  module Agent
     # Start a transfer using Aspera HTTP Gateway, using web socket secure for uploads
     # ref: https://api.ibm.com/explorer/catalog/aspera/product/ibm-aspera/api/http-gateway-api/doc/guides-toc
     # https://developer.ibm.com/apis/catalog?search=%22aspera%20http%22
@@ -25,7 +26,7 @@ module Aspera
     #   1     JSON.slice_upload   File start          "end_slice_upload"  sent_v2_delimiter
     #   2..   Binary              File binary chunks  "end upload"        sent_general
     #   last  JSON.slice_upload   File end            "end_slice_upload"  sent_v2_delimiter
-    class AgentHttpgw < Aspera::Fasp::AgentBase
+    class Httpgw < Base
       MSG_SEND_TRANSFER_SPEC = 'transfer_spec'
       MSG_SEND_SLICE_UPLOAD = 'slice_upload'
       MSG_RECV_DATA_RECEIVED_SIGNAL = 'end upload'
@@ -159,7 +160,7 @@ module Aspera
         # modify transfer spec to be suitable for GW
         transfer_spec['paths'].each do |item|
           # save actual file location to be able read contents later
-          file_to_add = FauxFile.open(item['source'])
+          file_to_add = Transfer::FauxFile.open(item['source'])
           if file_to_add
             item['source'] = file_to_add.path
             item['file_size'] = file_to_add.size
@@ -175,7 +176,8 @@ module Aspera
           files_to_read.push(file_to_add)
           total_bytes_to_transfer += item['file_size']
         end
-        upload_url = File.join(@gw_api.params[:base_url], @options[:api_version], 'upload')
+        # TODO: check that this is available in endpoints: @api_info['endpoints']
+        upload_url = File.join(@gw_base_url, @options[:api_version], 'upload')
         notify_progress(session_id: nil, type: :pre_start, info: 'connecting wss')
         # open web socket to end point (equivalent to Net::HTTP.start)
         http_session = Rest.start_http_session(upload_url)
@@ -185,7 +187,7 @@ module Aspera
         @ws_io.write(@ws_handshake.to_s)
         sleep(0.1)
         @ws_handshake << @ws_io.readuntil("\r\n\r\n")
-        raise 'Error in websocket handshake' unless @ws_handshake.finished?
+        Aspera.assert(@ws_handshake.finished?){'Error in websocket handshake'}
         Log.log.debug{"#{LOG_WS_SEND}handshake success"}
         # start read thread after handshake
         @ws_read_thread = Thread.new {process_read_thread}
@@ -226,7 +228,7 @@ module Aspera
             fileIndex:  file_index
           }
           file = files_to_read[file_index]
-          if file.is_a?(FauxFile)
+          if file.is_a?(Transfer::FauxFile)
             slice_info[:name] = file.path
           else
             file = File.open(file)
@@ -294,7 +296,7 @@ module Aspera
         end
         creation = @gw_api.create('v1/download', {'transfer_spec' => transfer_spec})[:data]
         transfer_uuid = creation['url'].split('/').last
-        file_dest =
+        file_name =
           if transfer_spec['zip_required'] || transfer_spec['paths'].length > 1
             # it is a zip file if zip is required or there is more than 1 file
             transfer_spec['download_name'] + '.zip'
@@ -302,8 +304,8 @@ module Aspera
             # it is a plain file if we don't require zip and there is only one file
             File.basename(transfer_spec['paths'].first['source'])
           end
-        file_dest = File.join(transfer_spec['destination_root'], file_dest)
-        @gw_api.call({operation: 'GET', subpath: "v1/download/#{transfer_uuid}", save_to_file: file_dest})
+        file_path = File.join(transfer_spec['destination_root'], file_name)
+        @gw_api.call(operation: 'GET', subpath: "v1/download/#{transfer_uuid}", save_to_file: file_path)
       end
 
       # start FASP transfer based on transfer spec (hash table)
@@ -311,14 +313,14 @@ module Aspera
       # HTTP download only supports file list
       def start_transfer(transfer_spec, token_regenerator: nil)
         raise 'GW URL must be set' if @gw_api.nil?
-        raise 'paths: must be Array' unless transfer_spec['paths'].is_a?(Array)
-        raise 'only token based transfer is supported in GW' unless transfer_spec['token'].is_a?(String)
+        Aspera.assert_type(transfer_spec['paths'], Array){'paths'}
+        Aspera.assert_type(transfer_spec['token'], String){'only token based transfer is supported in GW'}
         Log.log.debug{Log.dump(:user_spec, transfer_spec)}
         transfer_spec['authentication'] ||= 'token'
         case transfer_spec['direction']
-        when Fasp::TransferSpec::DIRECTION_SEND
+        when Transfer::Spec::DIRECTION_SEND
           upload(transfer_spec)
-        when Fasp::TransferSpec::DIRECTION_RECEIVE
+        when Transfer::Spec::DIRECTION_RECEIVE
           download(transfer_spec)
         else
           raise "unexpected direction: [#{transfer_spec['direction']}]"
@@ -339,10 +341,10 @@ module Aspera
 
       def initialize(opts)
         super(opts)
-        @options = AgentBase.options(default: DEFAULT_OPTIONS, options: opts)
+        @options = Base.options(default: DEFAULT_OPTIONS, options: opts)
         # remove /v1 from end of user-provided GW url: we need the base url only
-        @options[:url].gsub(%r{/v1/*$}, '')
-        @gw_api = Rest.new({base_url: @options[:url]})
+        @gw_base_url = @options[:url].gsub(%r{/v1/*$}, '')
+        @gw_api = Rest.new(base_url: @gw_base_url)
         @api_info = @gw_api.read('v1/info')[:data]
         Log.log.debug{Log.dump(:api_info, @api_info)}
         # web socket endpoint: by default use v2 (newer gateways), without base64 encoding

data/lib/aspera/{fasp/agent_node.rb → agent/node.rb}

@@ -1,17 +1,18 @@
 # frozen_string_literal: true
 
 # cspell:ignore precalc
-require 'aspera/fasp/agent_base'
-require 'aspera/fasp/transfer_spec'
-require 'aspera/node'
+require 'aspera/agent/base'
+require 'aspera/transfer/spec'
+require 'aspera/api/node'
 require 'aspera/log'
+require 'aspera/assert'
 require 'aspera/oauth'
 
 module Aspera
-  module Fasp
+  module Agent
     # this singleton class is used by the CLI to provide a common interface to start a transfer
     # before using it, the use must set the `node_api` member.
-    class AgentNode < Aspera::Fasp::AgentBase
+    class Node < Base
       DEFAULT_OPTIONS = {
         url:      :required,
         username: :required,
@@ -22,15 +23,15 @@ module Aspera
       # attr_writer :options
 
       def initialize(opts)
-        raise 'node specification must be Hash' unless opts.is_a?(Hash)
+        Aspera.assert_type(opts, Hash){'node agent options'}
         super(opts)
-        options = AgentBase.options(default: DEFAULT_OPTIONS, options: opts)
+        options = Base.options(default: DEFAULT_OPTIONS, options: opts)
         # root id is required for access key
         @root_id = options[:root_id]
         rest_params = { base_url: options[:url]}
-        if Oauth.bearer?(options[:password])
-          raise 'root_id is required for access key' if @root_id.nil?
-          rest_params[:headers] = Aspera::Node.bearer_headers(options[:password], access_key: options[:username])
+        if OAuth::Factory.bearer?(options[:password])
+          Aspera.assert(!@root_id.nil?){'root_id not allowed for access key'}
+          rest_params[:headers] = Api::Node.bearer_headers(options[:password], access_key: options[:username])
         else
           rest_params[:auth] = {
             type:     :basic,
@@ -38,7 +39,7 @@ module Aspera
             password: options[:password]
           }
         end
-        @node_api = Rest.new(rest_params)
+        @node_api = Rest.new(**rest_params)
         # TODO: currently only supports one transfer. This is bad shortcut. but ok for CLI.
         @transfer_id = nil
         # Log.log.debug{Log.dump(:agent_options, @options)}
@@ -46,11 +47,9 @@ module Aspera
 
       # used internally to ensure node api is set before using.
       def node_api_
-        raise StandardError, 'Before using this object, set the node_api attribute to a Aspera::Rest object' if @node_api.nil?
+        Aspera.assert(!@node_api.nil?){'Before using this object, set the node_api attribute to a Aspera::Rest object'}
         return @node_api
       end
-      # use this to read the node_api end point.
-      # attr_reader :node_api
 
       # use this to set the node_api end point before using the class.
       def node_api=(new_value)
@@ -65,30 +64,19 @@ module Aspera
         # add root id if access key
         if !@root_id.nil?
           case transfer_spec['direction']
-          when Fasp::TransferSpec::DIRECTION_SEND then transfer_spec['source_root_id'] = @root_id
-          when Fasp::TransferSpec::DIRECTION_RECEIVE then transfer_spec['destination_root_id'] = @root_id
-          else raise "unexpected direction in ts: #{transfer_spec['direction']}"
-          end
-        end
-        # manage special additional parameter
-        if transfer_spec.key?('EX_ssh_key_paths') && transfer_spec['EX_ssh_key_paths'].is_a?(Array) && !transfer_spec['EX_ssh_key_paths'].empty?
-          # not standard, so place standard field
-          if transfer_spec.key?('ssh_private_key')
-            Log.log.warn('Both ssh_private_key and EX_ssh_key_paths are present, using ssh_private_key')
-          else
-            Log.log.warn('EX_ssh_key_paths has multiple keys, using first one only') unless transfer_spec['EX_ssh_key_paths'].length.eql?(1)
-            transfer_spec['ssh_private_key'] = File.read(transfer_spec['EX_ssh_key_paths'].first)
-            transfer_spec.delete('EX_ssh_key_paths')
+          when Transfer::Spec::DIRECTION_SEND then transfer_spec['source_root_id'] = @root_id
+          when Transfer::Spec::DIRECTION_RECEIVE then transfer_spec['destination_root_id'] = @root_id
+          else Aspera.error_unexpected_value(transfer_spec['direction'])
           end
         end
         # add mandatory retry parameter for node api
         ts_tags = transfer_spec['tags']
-        if ts_tags.is_a?(Hash) && ts_tags[Fasp::TransferSpec::TAG_RESERVED].is_a?(Hash)
-          ts_tags[Fasp::TransferSpec::TAG_RESERVED]['xfer_retry'] ||= 150
+        if ts_tags.is_a?(Hash) && ts_tags[Transfer::Spec::TAG_RESERVED].is_a?(Hash)
+          ts_tags[Transfer::Spec::TAG_RESERVED]['xfer_retry'] ||= 150
         end
         # Optimization in case of sending to the same node
         # TODO: probably remove this, as /etc/hosts shall be used for that
-        if !transfer_spec['wss_enabled'] && transfer_spec['remote_host'].eql?(URI.parse(node_api_.params[:base_url]).host)
+        if !transfer_spec['wss_enabled'] && transfer_spec['remote_host'].eql?(URI.parse(node_api_.base_url).host)
           transfer_spec['remote_host'] = '127.0.0.1'
         end
         resp = node_api_.create('ops/transfers', transfer_spec)[:data]
@@ -132,10 +120,10 @@ module Aspera
             notify_progress(type: :end, session_id: @transfer_id)
             # Bug in HSTS ? transfer is marked failed, but there is no reason
             break if transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
-            raise Fasp::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
+            raise Transfer::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
           else
             Log.log.warn{"transfer_data -> #{transfer_data}"}
-            raise Fasp::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
+            raise Transfer::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
           end
           sleep(1.0)
         end