aspera-cli 4.19.0 → 4.21.1

data/lib/aspera/agent/direct.rb

@@ -72,7 +72,7 @@ module Aspera
         session = {
           id:                nil, # SessionId from INIT message in mgt port
           job_id:            SecureRandom.uuid, # job id (regroup sessions)
-          ts:                transfer_spec,     # transfer spec
+          ts:                transfer_spec,     # global transfer spec
           thread:            nil,               # Thread object monitoring management port, not nil when pushed to :sessions
           error:             nil,               # exception if failed
           io:                nil,               # management port server socket
@@ -84,7 +84,7 @@ module Aspera
         if multi_session_info.nil?
           Log.log.debug('Starting single session thread')
           # single session for transfer : simple
-          session[:thread] = Thread.new(session) {|session_info|transfer_thread_entry(session_info)}
+          session[:thread] = Thread.new {transfer_thread_entry(session)}
           @sessions.push(session)
         else
           Log.log.debug('Starting multi session threads')
@@ -101,7 +101,7 @@ module Aspera
             # option: increment (default as per ascp manual) or not (cluster on other side ?)
             args.unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @multi_incr_udp
             # finally start the thread
-            this_session[:thread] = Thread.new(this_session) {|session_info|transfer_thread_entry(session_info)}
+            this_session[:thread] = Thread.new {transfer_thread_entry(this_session)}
             @sessions.push(this_session)
           end
         end
@@ -132,18 +132,18 @@ module Aspera
 
       # @return [Array] list of sessions for a job
       def sessions_by_job(job_id)
-        @sessions.select{|session_info| session_info[:job_id].eql?(job_id)}
+        @sessions.select{|session| session[:job_id].eql?(job_id)}
       end
 
-      # This is the low level method to start the transfer process
+      # This is the low level method to start the transfer process.
+      # Typically started in a thread.
       # Start process with management port.
-      # returns
-      # raises FaspError on error
       # @param session this session information, keys :io and :token_regenerator
-      # @param env  [Hash]   environment variables
-      # @param name [Symbol] name of executable: :ascp, :ascp4 or :async
-      # @param args [Array]  command line arguments
+      # @param env  [Hash]   environment variables (comes from ascp_args)
+      # @param name [Symbol] name of executable: :ascp, :ascp4 or :async (comes from ascp_args)
+      # @param args [Array]  command line arguments (comes from ascp_args)
       # @return [nil] when process has exited
+      # @throw FaspError on error
       def start_and_monitor_process(
         session:,
         env:,
@@ -151,7 +151,7 @@ module Aspera
         args:
       )
         Aspera.assert_type(session, Hash)
-        notify_progress(session_id: nil, type: :pre_start, info: 'starting')
+        notify_progress(:pre_start, session_id: nil, info: 'starting')
         begin
           command_pid = nil
           # we use Socket directly, instead of TCPServer, as it gives access to lower level options
@@ -159,6 +159,8 @@ module Aspera
           mgt_server_socket = socket_class.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
           # open any available (0) local TCP port for use as management port
           mgt_server_socket.bind(Addrinfo.tcp(LISTEN_LOCAL_ADDRESS, SELECT_AVAILABLE_PORT))
+          # make port ready to accept connections, before starting ascp
+          mgt_server_socket.listen(1)
           # build arguments and add mgt port
           command_arguments = if name.eql?(:async)
             ["--exclusive-mgmt-port=#{mgt_server_socket.local_address.ip_port}"]
@@ -169,9 +171,9 @@ module Aspera
           # get location of command executable (ascp, async)
           command_path = Ascp::Installation.instance.path(name)
           command_pid = Environment.secure_spawn(env: env, exec: command_path, args: command_arguments)
-          notify_progress(session_id: nil, type: :pre_start, info: "waiting for #{name} to start")
-          mgt_server_socket.listen(1)
+          notify_progress(:pre_start, session_id: nil, info: "waiting for #{name} to start")
           # TODO: timeout does not work when Process.spawn is used... until process exits, then it works
+          # So we use select to detect that anything happens on the socket (connection)
           Log.log.debug{"before select, timeout: #{@spawn_timeout_sec}"}
           readable, _, _ = IO.select([mgt_server_socket], nil, nil, @spawn_timeout_sec)
           Log.log.debug('after select, before accept')
@@ -192,29 +194,30 @@ module Aspera
             next unless event
             # event is ready
             Log.log.trace1{Log.dump(:management_port, event)}
+            # store latest event by type
+            session[:id] = event['SessionId'] if event['Type'].eql?('INIT')
             @management_cb&.call(event)
             process_progress(event)
-            Log.log.error((event['Description']).to_s) if event['Type'].eql?('FILEERROR') # cspell:disable-line
+            Log.log.error(event['Description'].to_s) if event['Type'].eql?('FILEERROR') # cspell:disable-line
           end
           Log.log.debug('management io closed')
-          last_event = processor.last_event
           # check that last status was received before process exit
-          if last_event.is_a?(Hash)
-            case last_event['Type']
-            when 'ERROR'
-              if /bearer token/i.match?(last_event['Description']) &&
-                  session[:token_regenerator].respond_to?(:refreshed_transfer_token)
-                # regenerate token here, expired, or error on it
-                # Note: in multi-session, each session will have a different one.
-                Log.log.warn('Regenerating token for transfer')
-                env['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
-              end
-              raise Transfer::Error.new(last_event['Description'], last_event['Code'].to_i)
-            when 'DONE'
-              nil
-            else
-              raise "unexpected last event type: #{last_event['Type']}"
+          last_event = processor.last_event
+          raise Transfer::Error, "internal: no management event (#{last_event.class})" unless last_event.is_a?(Hash)
+          case last_event['Type']
+          when 'ERROR'
+            if /bearer token/i.match?(last_event['Description']) &&
+                session[:token_regenerator].respond_to?(:refreshed_transfer_token)
+              # regenerate token here, expired, or error on it
+              # Note: in multi-session, each session will have a different one.
+              Log.log.warn('Regenerating token for transfer')
+              env['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
             end
+            raise Transfer::Error.new(last_event['Description'], last_event['Code'].to_i)
+          when 'DONE'
+            nil
+          else
+            raise Transfer::Error, "unexpected last event type: #{last_event['Type']}, #{last_event['Description']}"
           end
         rescue SystemCallError => e
           # Process.spawn failed, or socket error
@@ -243,34 +246,37 @@ module Aspera
         nil
       end
 
+      attr_reader :sessions
+
       private
 
       # notify progress to callback
       # @param event management port event
+      # @param session sessin object
       def process_progress(event)
         session_id = event['SessionId']
         case event['Type']
         when 'INIT'
           @pre_calc_sent = false
           @pre_calc_last_size = nil
-          notify_progress(session_id: session_id, type: :session_start)
+          notify_progress(:session_start, session_id: session_id)
         when 'NOTIFICATION' # sent from remote
           if event.key?('PreTransferBytes')
             @pre_calc_sent = true
-            notify_progress(session_id: session_id, type: :session_size, info: event['PreTransferBytes'])
+            notify_progress(:session_size, session_id: session_id, info: event['PreTransferBytes'])
           end
         when 'STATS' # during transfer
           @pre_calc_last_size = event['TransferBytes'].to_i + event['StartByte'].to_i
-          notify_progress(session_id: session_id, type: :transfer, info: @pre_calc_last_size)
+          notify_progress(:transfer, session_id: session_id, info: @pre_calc_last_size)
         when 'DONE', 'ERROR' # end of session
           total_size = event['TransferBytes'].to_i + event['StartByte'].to_i
           if !@pre_calc_sent && !total_size.zero?
-            notify_progress(session_id: session_id, type: :session_size, info: total_size)
+            notify_progress(:session_size, session_id: session_id, info: total_size)
           end
           if @pre_calc_last_size != total_size
-            notify_progress(session_id: session_id, type: :transfer, info: total_size)
+            notify_progress(:transfer, session_id: session_id, info: total_size)
           end
-          notify_progress(session_id: session_id, type: :end)
+          notify_progress(:end, session_id: session_id)
           # cspell:disable
         when 'SESSION'
         when 'ARGSTOP'
@@ -283,14 +289,6 @@ module Aspera
         end
       end
 
-      # @return [Hash] session information
-      def session_by_id(id)
-        matches = @sessions.select{|session_info| session_info[:id].eql?(id)}
-        raise 'no such session' if matches.empty?
-        raise 'more than one session' if matches.length > 1
-        return matches.first
-      end
-
       # send command to management port of command (used in `asession)
       # @param job_id identified transfer process
       # @param session_index index of session (for multi session)
@@ -298,18 +296,10 @@ module Aspera
       # {'type'=>'START','source'=>_path_,'destination'=>_path_}
       # {'type'=>'DONE'}
       def send_command(job_id, data)
-        session = session_by_id(job_id)
+        session = @sessions.find{|session| session[:job_id].eql?(job_id)}
         Log.log.debug{"command: #{data}"}
-        # build command
-        command = data
-          .keys
-          .map{|k|"#{k.capitalize}: #{data[k]}"}
-          .unshift(MGT_HEADER)
-          .push('', '')
-          .join("\n")
-        session[:io].puts(command)
+        session[:io].puts(Ascp::Management.command_to_stream(data))
       end
-      attr_reader :sessions
 
       # options for initialize (same as values in option transfer_info)
       # @param ascp_args [Array] additional arguments to ascp

data/lib/aspera/agent/httpgw.rb

@@ -23,8 +23,7 @@ module Aspera
           @gw_api.upload(transfer_spec)
         when Transfer::Spec::DIRECTION_RECEIVE
           @gw_api.download(transfer_spec)
-        else
-          raise "unexpected direction: [#{transfer_spec['direction']}]"
+        else Aspera.error_unexpected_value(transfer_spec['direction']){'direction'}
         end
       end
 
@@ -54,7 +53,7 @@ module Aspera
           api_version:       api_version,
           upload_chunk_size: upload_chunk_size,
           synchronous:       synchronous,
-          notify_cb:         ->(**event) { notify_progress(**event) }
+          notify_cb:         ->(*pa, **ka) { notify_progress(*pa, **ka) }
         )
       end
     end

data/lib/aspera/agent/node.rb

@@ -50,14 +50,6 @@ module Aspera
         return @node_api
       end
 
-      # use this to set the node_api end point before using the class.
-      def node_api=(new_value)
-        if !@node_api.nil? && !new_value.nil?
-          Log.log.warn('overriding existing node api value')
-        end
-        @node_api = new_value
-      end
-
       # generic method
       def start_transfer(transfer_spec, token_regenerator: nil)
         # add root id if access key
@@ -78,7 +70,7 @@ module Aspera
         if !transfer_spec['wss_enabled'] && transfer_spec['remote_host'].eql?(URI.parse(node_api_.base_url).host)
           transfer_spec['remote_host'] = '127.0.0.1'
         end
-        resp = node_api_.create('ops/transfers', transfer_spec)[:data]
+        resp = node_api_.create('ops/transfers', transfer_spec)
         @transfer_id = resp['id']
         Log.log.debug{"tr_id=#{@transfer_id}"}
         return @transfer_id
@@ -92,37 +84,35 @@ module Aspera
         # lets emulate management events to display progress bar
         loop do
           # status is empty sometimes with status 200...
-          transfer_data = node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {'status' => 'unknown'} rescue {'status' => 'waiting(api error)'}
+          transfer_data = node_api_.read("ops/transfers/#{@transfer_id}") || {'status' => 'unknown'} rescue {'status' => 'waiting(api error)'}
           case transfer_data['status']
           when 'waiting', 'partially_completed', 'unknown', 'waiting(read error)'
-            notify_progress(session_id: nil, type: :pre_start, info: transfer_data['status'])
+            notify_progress(:pre_start, session_id: nil, info: transfer_data['status'])
           when 'running'
             if !session_started
-              notify_progress(session_id: @transfer_id, type: :session_start)
+              notify_progress(:session_start, session_id: @transfer_id)
               session_started = true
             end
             message = transfer_data['status']
             message = "#{message} (#{transfer_data['error_desc']})" if !transfer_data['error_desc']&.empty?
-            notify_progress(session_id: nil, type: :pre_start, info: message)
+            notify_progress(:pre_start, session_id: nil, info: message)
             if bytes_expected.nil? &&
                 transfer_data['precalc'].is_a?(Hash) &&
                 transfer_data['precalc']['status'].eql?('ready')
               bytes_expected = transfer_data['precalc']['bytes_expected']
-              notify_progress(type: :session_size, session_id: @transfer_id, info: bytes_expected)
+              notify_progress(:session_size, session_id: @transfer_id, info: bytes_expected)
             end
-            notify_progress(type: :transfer, session_id: @transfer_id, info: transfer_data['bytes_transferred'])
+            notify_progress(:transfer, session_id: @transfer_id, info: transfer_data['bytes_transferred'])
           when 'completed'
-            notify_progress(type: :transfer, session_id: @transfer_id, info: bytes_expected) if bytes_expected
-            notify_progress(type: :end, session_id: @transfer_id)
+            notify_progress(:transfer, session_id: @transfer_id, info: bytes_expected) if bytes_expected
+            notify_progress(:end, session_id: @transfer_id)
             break
           when 'failed'
-            notify_progress(type: :end, session_id: @transfer_id)
+            notify_progress(:end, session_id: @transfer_id)
             # Bug in HSTS ? transfer is marked failed, but there is no reason
             break if transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
             raise Transfer::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
-          else
-            Log.log.warn{"transfer_data -> #{transfer_data}"}
-            raise Transfer::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
+          else Aspera.error_unexpected_value(transfer_data['status']){"transfer_data -> #{transfer_data}"}
           end
           sleep(1.0)
         end

data/lib/aspera/agent/{trsdk.rb → transferd.rb}

@@ -1,41 +1,21 @@
 # frozen_string_literal: true
 
 require 'aspera/agent/base'
-require 'aspera/ascp/installation'
+require 'aspera/products/transferd'
 require 'aspera/temp_file_manager'
-require 'aspera/log'
-require 'aspera/assert'
 require 'json'
 require 'uri'
+require 'transferd_services_pb'
 
 module Aspera
   module Agent
-    class Trsdk < Base
-      # see https://github.com/grpc/grpc/blob/master/doc/naming.md
+    class Transferd < Base
+      # https://github.com/grpc/grpc/blob/master/doc/naming.md
       # https://grpc.io/docs/guides/custom-name-resolution/
       LOCAL_SOCKET_ADDR = '127.0.0.1'
       PORT_SEP = ':'
       # port zero means select a random available high port
       AUTO_LOCAL_TCP_PORT = "#{PORT_SEP}0"
-      class << self
-        # Well, the port number is only in log file
-        def daemon_port_from_log(log_file)
-          result = nil
-          # if port is zero, a dynamic port was created, get it
-          File.open(log_file, 'r') do |file|
-            file.each_line do |line|
-              # Well, it's tricky to depend on log
-              if (m = line.match(/Info: API Server: Listening on ([^:]+):(\d+) /))
-                result = m[2].to_i
-                # no "break" , need to read last matching log line
-              end
-            end
-          end
-          raise 'Port not found in daemon logs' if result.nil?
-          Log.log.debug{"Got port #{result} from log"}
-          return result
-        end
-      end
 
       # @param url [String] URL of the transfer manager daemon
       # @param external [Boolean] if true, expect that an external daemon is already running
@@ -48,34 +28,30 @@ module Aspera
         **base_options
       )
         super(**base_options)
-        is_local_auto_port = @url.eql?(AUTO_LOCAL_TCP_PORT)
-        raise 'Cannot use options `keep` or `external` with port zero' if is_local_auto_port && (@keep || @external)
-        # load SDK stub class on demand, as it's an optional gem
-        $LOAD_PATH.unshift(Ascp::Installation.instance.sdk_ruby_folder)
-        require 'transfer_services_pb'
+        @keep = keep
+        is_local_auto_port = url.eql?(AUTO_LOCAL_TCP_PORT)
+        raise 'Cannot use options `keep` or `external` with port zero' if is_local_auto_port && (@keep || external)
         # keep PID for optional shutdown
         @daemon_pid = nil
-        daemon_endpoint = @url
+        daemon_endpoint = url
         Log.log.debug{Log.dump(:daemon_endpoint, daemon_endpoint)}
         # retry loop
         begin
           # no address: local bind
           daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{daemon_endpoint}" if daemon_endpoint.match?(/^#{PORT_SEP}[0-9]+$/o)
           # Create stub (without credentials)
-          @transfer_client = Transfersdk::TransferService::Stub.new(daemon_endpoint, :this_channel_is_insecure)
+          @transfer_client = ::Transferd::Api::TransferService::Stub.new(daemon_endpoint, :this_channel_is_insecure)
           # Initiate actual connection
-          get_info_response = @transfer_client.get_info(Transfersdk::InstanceInfoRequest.new)
+          get_info_response = @transfer_client.get_info(::Transferd::Api::InstanceInfoRequest.new)
           Log.log.debug{"Daemon info: #{get_info_response}"}
-          Log.log.warn{'Attached to existing daemon'} unless @daemon_pid || @external || @keep
+          Log.log.warn{'Attached to existing daemon'} unless @daemon_pid || external || @keep
           at_exit{shutdown}
         rescue GRPC::Unavailable => e
           # if transferd is external: do not start it, or other error
-          raise if @external || !e.message.include?('failed to connect')
+          raise if external || !e.message.include?('failed to connect')
           # we already tried to start a daemon, but it failed
           Aspera.assert(@daemon_pid.nil?){"Daemon started with PID #{@daemon_pid}, but connection failed to #{daemon_endpoint}}"}
-          Log.log.warn('no daemon present, starting daemon...') if @external
-          # location of daemon binary
-          sdk_folder = File.realpath(File.join(Ascp::Installation.instance.sdk_ruby_folder, '..'))
+          Log.log.warn('no daemon present, starting daemon...') if external
           # transferd only supports local ip and port
           daemon_uri = URI.parse("ipv4://#{daemon_endpoint}")
           Aspera.assert(daemon_uri.scheme.eql?('ipv4')){"Invalid scheme daemon URI #{daemon_endpoint}"}
@@ -86,8 +62,8 @@ module Aspera
             fasp_runtime: {
               use_embedded: false,
               user_defined: {
-                bin: sdk_folder,
-                etc: sdk_folder
+                bin: Products::Transferd.sdk_directory,
+                etc: Products::Transferd.sdk_directory
               }
             }
           }
@@ -113,7 +89,7 @@ module Aspera
           Process.detach(@daemon_pid) if @keep
           at_exit {shutdown}
           # update port for next connection attempt (if auto high port was requested)
-          daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{PORT_SEP}#{self.class.daemon_port_from_log(log_stdout)}" if is_local_auto_port
+          daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{PORT_SEP}#{Products::Transferd.daemon_port_from_log(log_stdout)}" if is_local_auto_port
           # local daemon started, try again
           retry
         end
@@ -121,9 +97,9 @@ module Aspera
 
       def start_transfer(transfer_spec, token_regenerator: nil)
         # create a transfer request
-        transfer_request = Transfersdk::TransferRequest.new(
-          transferType: Transfersdk::TransferType::FILE_REGULAR, # transfer type (file/stream)
-          config: Transfersdk::TransferConfig.new, # transfer configuration
+        transfer_request = ::Transferd::Api::TransferRequest.new(
+          transferType: ::Transferd::Api::TransferType::FILE_REGULAR, # transfer type (file/stream)
+          config: ::Transferd::Api::TransferConfig.new, # transfer configuration
           transferSpec: transfer_spec.to_json) # transfer definition
         # send start transfer request to the transfer manager daemon
         start_transfer_response = @transfer_client.start_transfer(transfer_request)
@@ -137,30 +113,30 @@ module Aspera
         session_started = false
         bytes_expected = nil
         # monitor transfer status
-        @transfer_client.monitor_transfers(Transfersdk::RegistrationRequest.new(transferId: [@transfer_id])) do |response|
+        @transfer_client.monitor_transfers(::Transferd::Api::RegistrationRequest.new(transferId: [@transfer_id])) do |response|
           Log.log.debug{Log.dump(:response, response.to_h)}
           # Log.log.debug{"#{response.sessionInfo.preTransferBytes} #{response.transferInfo.bytesTransferred}"}
           case response.status
           when :RUNNING
             if !session_started
-              notify_progress(session_id: @transfer_id, type: :session_start)
+              notify_progress(:session_start, session_id: @transfer_id)
               session_started = true
             end
             if bytes_expected.nil? &&
                 !response.sessionInfo.preTransferBytes.eql?(0)
               bytes_expected = response.sessionInfo.preTransferBytes
-              notify_progress(type: :session_size, session_id: @transfer_id, info: bytes_expected)
+              notify_progress(:session_size, session_id: @transfer_id, info: bytes_expected)
             end
-            notify_progress(type: :transfer, session_id: @transfer_id, info: response.transferInfo.bytesTransferred)
+            notify_progress(:transfer, session_id: @transfer_id, info: response.transferInfo.bytesTransferred)
           when :COMPLETED
-            notify_progress(type: :transfer, session_id: @transfer_id, info: bytes_expected) if bytes_expected
-            notify_progress(type: :end, session_id: @transfer_id)
+            notify_progress(:transfer, session_id: @transfer_id, info: bytes_expected) if bytes_expected
+            notify_progress(:end, session_id: @transfer_id)
             break
           when :FAILED, :CANCELED
-            notify_progress(type: :end, session_id: @transfer_id)
+            notify_progress(:end, session_id: @transfer_id)
             raise Transfer::Error, JSON.parse(response.message)['Description']
           when :QUEUED, :UNKNOWN_STATUS, :PAUSED, :ORPHANED
-            notify_progress(session_id: nil, type: :pre_start, info: response.status.to_s.downcase)
+            notify_progress(:pre_start, session_id: nil, info: response.status.to_s.downcase)
           else
             Log.log.error{"unknown status#{response.status}"}
           end

data/lib/aspera/api/alee.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'aspera/api/aoc.rb'
+module Aspera
+  module Api
+    class Alee < Aspera::Rest
+      def initialize(entitlement_id, customer_id, api_domain: AoC::SAAS_DOMAIN_PROD, version: 'v1')
+        super(
+          base_url: "https://api.#{api_domain}/metering/#{version}",
+          headers:  {'X-Aspera-Entitlement-Authorization' => Rest.basic_token(entitlement_id, customer_id)}
+        )
+      end
+    end
+  end
+end