aspera-cli 4.16.0 → 4.17.0

data/lib/aspera/{fasp/agent_httpgw.rb → agent/httpgw.rb}

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require 'aspera/fasp/agent_base'
-require 'aspera/fasp/transfer_spec'
-require 'aspera/fasp/faux_file'
+require 'aspera/agent/base'
+require 'aspera/transfer/spec'
+require 'aspera/transfer/faux_file'
 require 'aspera/log'
 require 'aspera/assert'
 require 'aspera/rest'
@@ -12,7 +12,7 @@ require 'base64'
 require 'json'
 
 module Aspera
-  module Fasp
+  module Agent
     # Start a transfer using Aspera HTTP Gateway, using web socket secure for uploads
     # ref: https://api.ibm.com/explorer/catalog/aspera/product/ibm-aspera/api/http-gateway-api/doc/guides-toc
     # https://developer.ibm.com/apis/catalog?search=%22aspera%20http%22
@@ -26,7 +26,7 @@ module Aspera
     #   1     JSON.slice_upload   File start          "end_slice_upload"  sent_v2_delimiter
     #   2..   Binary              File binary chunks  "end upload"        sent_general
     #   last  JSON.slice_upload   File end            "end_slice_upload"  sent_v2_delimiter
-    class AgentHttpgw < Aspera::Fasp::AgentBase
+    class Httpgw < Base
       MSG_SEND_TRANSFER_SPEC = 'transfer_spec'
       MSG_SEND_SLICE_UPLOAD = 'slice_upload'
       MSG_RECV_DATA_RECEIVED_SIGNAL = 'end upload'
@@ -160,7 +160,7 @@ module Aspera
         # modify transfer spec to be suitable for GW
         transfer_spec['paths'].each do |item|
           # save actual file location to be able read contents later
-          file_to_add = FauxFile.open(item['source'])
+          file_to_add = Transfer::FauxFile.open(item['source'])
           if file_to_add
             item['source'] = file_to_add.path
             item['file_size'] = file_to_add.size
@@ -176,7 +176,8 @@ module Aspera
           files_to_read.push(file_to_add)
           total_bytes_to_transfer += item['file_size']
         end
-        upload_url = File.join(@gw_api.params[:base_url], @options[:api_version], 'upload')
+        # TODO: check that this is available in endpoints: @api_info['endpoints']
+        upload_url = File.join(@gw_base_url, @options[:api_version], 'upload')
         notify_progress(session_id: nil, type: :pre_start, info: 'connecting wss')
         # open web socket to end point (equivalent to Net::HTTP.start)
         http_session = Rest.start_http_session(upload_url)
@@ -186,7 +187,7 @@ module Aspera
         @ws_io.write(@ws_handshake.to_s)
         sleep(0.1)
         @ws_handshake << @ws_io.readuntil("\r\n\r\n")
-        assert(@ws_handshake.finished?){'Error in websocket handshake'}
+        Aspera.assert(@ws_handshake.finished?){'Error in websocket handshake'}
         Log.log.debug{"#{LOG_WS_SEND}handshake success"}
         # start read thread after handshake
         @ws_read_thread = Thread.new {process_read_thread}
@@ -227,7 +228,7 @@ module Aspera
             fileIndex:  file_index
           }
           file = files_to_read[file_index]
-          if file.is_a?(FauxFile)
+          if file.is_a?(Transfer::FauxFile)
             slice_info[:name] = file.path
           else
             file = File.open(file)
@@ -295,7 +296,7 @@ module Aspera
         end
         creation = @gw_api.create('v1/download', {'transfer_spec' => transfer_spec})[:data]
         transfer_uuid = creation['url'].split('/').last
-        file_dest =
+        file_name =
           if transfer_spec['zip_required'] || transfer_spec['paths'].length > 1
             # it is a zip file if zip is required or there is more than 1 file
             transfer_spec['download_name'] + '.zip'
@@ -303,8 +304,8 @@ module Aspera
             # it is a plain file if we don't require zip and there is only one file
             File.basename(transfer_spec['paths'].first['source'])
           end
-        file_dest = File.join(transfer_spec['destination_root'], file_dest)
-        @gw_api.call({operation: 'GET', subpath: "v1/download/#{transfer_uuid}", save_to_file: file_dest})
+        file_path = File.join(transfer_spec['destination_root'], file_name)
+        @gw_api.call(operation: 'GET', subpath: "v1/download/#{transfer_uuid}", save_to_file: file_path)
       end
 
       # start FASP transfer based on transfer spec (hash table)
@@ -312,14 +313,14 @@ module Aspera
       # HTTP download only supports file list
       def start_transfer(transfer_spec, token_regenerator: nil)
         raise 'GW URL must be set' if @gw_api.nil?
-        assert_type(transfer_spec['paths'], Array){'paths'}
-        assert_type(transfer_spec['token'], String){'only token based transfer is supported in GW'}
+        Aspera.assert_type(transfer_spec['paths'], Array){'paths'}
+        Aspera.assert_type(transfer_spec['token'], String){'only token based transfer is supported in GW'}
         Log.log.debug{Log.dump(:user_spec, transfer_spec)}
         transfer_spec['authentication'] ||= 'token'
         case transfer_spec['direction']
-        when Fasp::TransferSpec::DIRECTION_SEND
+        when Transfer::Spec::DIRECTION_SEND
           upload(transfer_spec)
-        when Fasp::TransferSpec::DIRECTION_RECEIVE
+        when Transfer::Spec::DIRECTION_RECEIVE
           download(transfer_spec)
         else
           raise "unexpected direction: [#{transfer_spec['direction']}]"
@@ -340,10 +341,10 @@ module Aspera
 
       def initialize(opts)
         super(opts)
-        @options = AgentBase.options(default: DEFAULT_OPTIONS, options: opts)
+        @options = Base.options(default: DEFAULT_OPTIONS, options: opts)
         # remove /v1 from end of user-provided GW url: we need the base url only
-        @options[:url].gsub(%r{/v1/*$}, '')
-        @gw_api = Rest.new({base_url: @options[:url]})
+        @gw_base_url = @options[:url].gsub(%r{/v1/*$}, '')
+        @gw_api = Rest.new(base_url: @gw_base_url)
         @api_info = @gw_api.read('v1/info')[:data]
         Log.log.debug{Log.dump(:api_info, @api_info)}
         # web socket endpoint: by default use v2 (newer gateways), without base64 encoding

data/lib/aspera/{fasp/agent_node.rb → agent/node.rb}

@@ -1,18 +1,18 @@
 # frozen_string_literal: true
 
 # cspell:ignore precalc
-require 'aspera/fasp/agent_base'
-require 'aspera/fasp/transfer_spec'
-require 'aspera/node'
+require 'aspera/agent/base'
+require 'aspera/transfer/spec'
+require 'aspera/api/node'
 require 'aspera/log'
 require 'aspera/assert'
 require 'aspera/oauth'
 
 module Aspera
-  module Fasp
+  module Agent
     # this singleton class is used by the CLI to provide a common interface to start a transfer
     # before using it, the use must set the `node_api` member.
-    class AgentNode < Aspera::Fasp::AgentBase
+    class Node < Base
       DEFAULT_OPTIONS = {
         url:      :required,
         username: :required,
@@ -23,15 +23,15 @@ module Aspera
       # attr_writer :options
 
       def initialize(opts)
-        assert_type(opts, Hash){'node agent options'}
+        Aspera.assert_type(opts, Hash){'node agent options'}
         super(opts)
-        options = AgentBase.options(default: DEFAULT_OPTIONS, options: opts)
+        options = Base.options(default: DEFAULT_OPTIONS, options: opts)
         # root id is required for access key
         @root_id = options[:root_id]
         rest_params = { base_url: options[:url]}
-        if Oauth.bearer?(options[:password])
-          raise 'root_id is required for access key' if @root_id.nil?
-          rest_params[:headers] = Aspera::Node.bearer_headers(options[:password], access_key: options[:username])
+        if OAuth::Factory.bearer?(options[:password])
+          Aspera.assert(!@root_id.nil?){'root_id not allowed for access key'}
+          rest_params[:headers] = Api::Node.bearer_headers(options[:password], access_key: options[:username])
         else
           rest_params[:auth] = {
             type:     :basic,
@@ -39,7 +39,7 @@ module Aspera
             password: options[:password]
           }
         end
-        @node_api = Rest.new(rest_params)
+        @node_api = Rest.new(**rest_params)
         # TODO: currently only supports one transfer. This is bad shortcut. but ok for CLI.
         @transfer_id = nil
         # Log.log.debug{Log.dump(:agent_options, @options)}
@@ -47,11 +47,9 @@ module Aspera
 
       # used internally to ensure node api is set before using.
       def node_api_
-        raise StandardError, 'Before using this object, set the node_api attribute to a Aspera::Rest object' if @node_api.nil?
+        Aspera.assert(!@node_api.nil?){'Before using this object, set the node_api attribute to a Aspera::Rest object'}
         return @node_api
       end
-      # use this to read the node_api end point.
-      # attr_reader :node_api
 
       # use this to set the node_api end point before using the class.
       def node_api=(new_value)
@@ -66,30 +64,19 @@ module Aspera
         # add root id if access key
         if !@root_id.nil?
           case transfer_spec['direction']
-          when Fasp::TransferSpec::DIRECTION_SEND then transfer_spec['source_root_id'] = @root_id
-          when Fasp::TransferSpec::DIRECTION_RECEIVE then transfer_spec['destination_root_id'] = @root_id
-          else error_unexpected_value(transfer_spec['direction'])
-          end
-        end
-        # manage special additional parameter
-        if transfer_spec.key?('EX_ssh_key_paths') && transfer_spec['EX_ssh_key_paths'].is_a?(Array) && !transfer_spec['EX_ssh_key_paths'].empty?
-          # not standard, so place standard field
-          if transfer_spec.key?('ssh_private_key')
-            Log.log.warn('Both ssh_private_key and EX_ssh_key_paths are present, using ssh_private_key')
-          else
-            Log.log.warn('EX_ssh_key_paths has multiple keys, using first one only') unless transfer_spec['EX_ssh_key_paths'].length.eql?(1)
-            transfer_spec['ssh_private_key'] = File.read(transfer_spec['EX_ssh_key_paths'].first)
-            transfer_spec.delete('EX_ssh_key_paths')
+          when Transfer::Spec::DIRECTION_SEND then transfer_spec['source_root_id'] = @root_id
+          when Transfer::Spec::DIRECTION_RECEIVE then transfer_spec['destination_root_id'] = @root_id
+          else Aspera.error_unexpected_value(transfer_spec['direction'])
           end
         end
         # add mandatory retry parameter for node api
         ts_tags = transfer_spec['tags']
-        if ts_tags.is_a?(Hash) && ts_tags[Fasp::TransferSpec::TAG_RESERVED].is_a?(Hash)
-          ts_tags[Fasp::TransferSpec::TAG_RESERVED]['xfer_retry'] ||= 150
+        if ts_tags.is_a?(Hash) && ts_tags[Transfer::Spec::TAG_RESERVED].is_a?(Hash)
+          ts_tags[Transfer::Spec::TAG_RESERVED]['xfer_retry'] ||= 150
         end
         # Optimization in case of sending to the same node
         # TODO: probably remove this, as /etc/hosts shall be used for that
-        if !transfer_spec['wss_enabled'] && transfer_spec['remote_host'].eql?(URI.parse(node_api_.params[:base_url]).host)
+        if !transfer_spec['wss_enabled'] && transfer_spec['remote_host'].eql?(URI.parse(node_api_.base_url).host)
           transfer_spec['remote_host'] = '127.0.0.1'
         end
         resp = node_api_.create('ops/transfers', transfer_spec)[:data]
@@ -133,10 +120,10 @@ module Aspera
             notify_progress(type: :end, session_id: @transfer_id)
             # Bug in HSTS ? transfer is marked failed, but there is no reason
             break if transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
-            raise Fasp::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
+            raise Transfer::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
           else
             Log.log.warn{"transfer_data -> #{transfer_data}"}
-            raise Fasp::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
+            raise Transfer::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
           end
           sleep(1.0)
         end

data/lib/aspera/{fasp/agent_trsdk.rb → agent/trsdk.rb}

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require 'aspera/fasp/agent_base'
-require 'aspera/fasp/installation'
+require 'aspera/agent/base'
+require 'aspera/ascp/installation'
 require 'aspera/temp_file_manager'
 require 'aspera/log'
 require 'aspera/assert'
@@ -9,8 +9,8 @@ require 'json'
 require 'uri'
 
 module Aspera
-  module Fasp
-    class AgentTrsdk < Aspera::Fasp::AgentBase
+  module Agent
+    class Trsdk < Base
       # see https://github.com/grpc/grpc/blob/master/doc/naming.md
       # https://grpc.io/docs/guides/custom-name-resolution/
       LOCAL_SOCKET_ADDR = '127.0.0.1'
@@ -47,12 +47,12 @@ module Aspera
       # options come from transfer_info
       def initialize(user_opts={})
         super(user_opts)
-        @options = AgentBase.options(default: DEFAULT_OPTIONS, options: user_opts)
+        @options = Base.options(default: DEFAULT_OPTIONS, options: user_opts)
         is_local_auto_port = @options[:url].eql?(AUTO_LOCAL_TCP_PORT)
         raise 'Cannot use options `keep` or `external` with port zero' if is_local_auto_port && (@options[:keep] || @options[:external])
         Log.log.debug{Log.dump(:agent_options, @options)}
         # load SDK stub class on demand, as it's an optional gem
-        $LOAD_PATH.unshift(Installation.instance.sdk_ruby_folder)
+        $LOAD_PATH.unshift(Ascp::Installation.instance.sdk_ruby_folder)
         require 'transfer_services_pb'
         # keep PID for optional shutdown
         @daemon_pid = nil
@@ -73,13 +73,13 @@ module Aspera
           # if transferd is external: do not start it, or other error
           raise if @options[:external] || !e.message.include?('failed to connect')
           # we already tried to start a daemon, but it failed
-          assert(@daemon_pid.nil?){"Daemon started with PID #{@daemon_pid}, but connection failed to #{daemon_endpoint}}"}
+          Aspera.assert(@daemon_pid.nil?){"Daemon started with PID #{@daemon_pid}, but connection failed to #{daemon_endpoint}}"}
           Log.log.warn('no daemon present, starting daemon...') if @options[:external]
           # location of daemon binary
-          sdk_folder = File.realpath(File.join(Installation.instance.sdk_ruby_folder, '..'))
+          sdk_folder = File.realpath(File.join(Ascp::Installation.instance.sdk_ruby_folder, '..'))
           # transferd only supports local ip and port
           daemon_uri = URI.parse("ipv4://#{daemon_endpoint}")
-          assert(daemon_uri.scheme.eql?('ipv4')){"Invalid scheme daemon URI #{daemon_endpoint}"}
+          Aspera.assert(daemon_uri.scheme.eql?('ipv4')){"Invalid scheme daemon URI #{daemon_endpoint}"}
           # create a config file for daemon
           config = {
             address:      daemon_uri.host,
@@ -99,7 +99,7 @@ module Aspera
           log_stdout = "#{transferd_base_tmp}.out"
           log_stderr = "#{transferd_base_tmp}.err"
           File.write(conf_file, config.to_json)
-          @daemon_pid = Process.spawn(Installation.instance.path(:transferd), '--config', conf_file, out: log_stdout, err: log_stderr)
+          @daemon_pid = Process.spawn(Ascp::Installation.instance.path(:transferd), '--config', conf_file, out: log_stdout, err: log_stderr)
           begin
             # wait for process to initialize, max 2 seconds
             Timeout.timeout(2.0) do
@@ -159,7 +159,7 @@ module Aspera
             break
           when :FAILED, :CANCELED
             notify_progress(type: :end, session_id: @transfer_id)
-            raise Fasp::Error, JSON.parse(response.message)['Description']
+            raise Transfer::Error, JSON.parse(response.message)['Description']
           when :QUEUED, :UNKNOWN_STATUS, :PAUSED, :ORPHANED
             notify_progress(session_id: nil, type: :pre_start, info: response.status.to_s.downcase)
           else