aspera-cli 4.14.0 → 4.16.0

data/lib/aspera/fasp/agent_node.rb

@@ -1,32 +1,37 @@
 # frozen_string_literal: true
 
+# cspell:ignore precalc
 require 'aspera/fasp/agent_base'
 require 'aspera/fasp/transfer_spec'
 require 'aspera/node'
 require 'aspera/log'
-require 'tty-spinner'
+require 'aspera/assert'
+require 'aspera/oauth'
 
 module Aspera
   module Fasp
     # this singleton class is used by the CLI to provide a common interface to start a transfer
     # before using it, the use must set the `node_api` member.
     class AgentNode < Aspera::Fasp::AgentBase
+      DEFAULT_OPTIONS = {
+        url:      :required,
+        username: :required,
+        password: :required,
+        root_id:  nil
+      }.freeze
       # option include: root_id if the node is an access key
-      attr_writer :options
+      # attr_writer :options
 
-      def initialize(options)
-        raise 'node specification must be Hash' unless options.is_a?(Hash)
-        %i[url username password].each { |k| raise "missing parameter [#{k}] in node specification: #{options}" unless options.key?(k) }
-        super()
+      def initialize(opts)
+        assert_type(opts, Hash){'node agent options'}
+        super(opts)
+        options = AgentBase.options(default: DEFAULT_OPTIONS, options: opts)
         # root id is required for access key
         @root_id = options[:root_id]
         rest_params = { base_url: options[:url]}
-        if /^Bearer /.match?(options[:password])
-          rest_params[:headers] = {
-            Aspera::Node::HEADER_X_ASPERA_ACCESS_KEY => options[:username],
-            'Authorization'                          => options[:password]
-          }
+        if Oauth.bearer?(options[:password])
           raise 'root_id is required for access key' if @root_id.nil?
+          rest_params[:headers] = Aspera::Node.bearer_headers(options[:password], access_key: options[:username])
         else
           rest_params[:auth] = {
             type:     :basic,
@@ -37,6 +42,7 @@ module Aspera
         @node_api = Rest.new(rest_params)
         # TODO: currently only supports one transfer. This is bad shortcut. but ok for CLI.
         @transfer_id = nil
+        # Log.log.debug{Log.dump(:agent_options, @options)}
       end
 
       # used internally to ensure node api is set before using.
@@ -45,7 +51,7 @@ module Aspera
         return @node_api
       end
       # use this to read the node_api end point.
-      attr_reader :node_api
+      # attr_reader :node_api
 
       # use this to set the node_api end point before using the class.
       def node_api=(new_value)
@@ -62,7 +68,7 @@ module Aspera
           case transfer_spec['direction']
           when Fasp::TransferSpec::DIRECTION_SEND then transfer_spec['source_root_id'] = @root_id
           when Fasp::TransferSpec::DIRECTION_RECEIVE then transfer_spec['destination_root_id'] = @root_id
-          else raise "unexpected direction in ts: #{transfer_spec['direction']}"
+          else error_unexpected_value(transfer_spec['direction'])
           end
         end
         # manage special additional parameter
@@ -94,43 +100,45 @@ module Aspera
 
       # generic method
       def wait_for_transfers_completion
-        started = false
-        spinner = nil
+        # set to true when we know the total size of the transfer
+        session_started = false
+        bytes_expected = nil
         # lets emulate management events to display progress bar
         loop do
           # status is empty sometimes with status 200...
-          transfer_data = node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {'status' => 'unknown'} rescue {'status' => 'waiting(read error)'}
+          transfer_data = node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {'status' => 'unknown'} rescue {'status' => 'waiting(api error)'}
           case transfer_data['status']
-          when 'completed'
-            notify_end(@transfer_id)
-            break
           when 'waiting', 'partially_completed', 'unknown', 'waiting(read error)'
-            if spinner.nil?
-              spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
-              spinner.start
-            end
-            spinner.update(title: transfer_data['status'])
-            spinner.spin
+            notify_progress(session_id: nil, type: :pre_start, info: transfer_data['status'])
           when 'running'
-            if !started && transfer_data['precalc'].is_a?(Hash) &&
+            if !session_started
+              notify_progress(session_id: @transfer_id, type: :session_start)
+              session_started = true
+            end
+            message = transfer_data['status']
+            message = "#{message} (#{transfer_data['error_desc']})" if !transfer_data['error_desc']&.empty?
+            notify_progress(session_id: nil, type: :pre_start, info: message)
+            if bytes_expected.nil? &&
+                transfer_data['precalc'].is_a?(Hash) &&
                 transfer_data['precalc']['status'].eql?('ready')
-              notify_begin(@transfer_id, transfer_data['precalc']['bytes_expected'])
-              started = true
-            else
-              notify_progress(@transfer_id, transfer_data['bytes_transferred'])
+              bytes_expected = transfer_data['precalc']['bytes_expected']
+              notify_progress(type: :session_size, session_id: @transfer_id, info: bytes_expected)
             end
+            notify_progress(type: :transfer, session_id: @transfer_id, info: transfer_data['bytes_transferred'])
+          when 'completed'
+            notify_progress(type: :transfer, session_id: @transfer_id, info: bytes_expected) if bytes_expected
+            notify_progress(type: :end, session_id: @transfer_id)
+            break
           when 'failed'
+            notify_progress(type: :end, session_id: @transfer_id)
             # Bug in HSTS ? transfer is marked failed, but there is no reason
-            if transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
-              notify_end(@transfer_id)
-              break
-            end
+            break if transfer_data['error_code'].eql?(0) && transfer_data['error_desc'].empty?
             raise Fasp::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
           else
             Log.log.warn{"transfer_data -> #{transfer_data}"}
             raise Fasp::Error, "status: #{transfer_data['status']}. code: #{transfer_data['error_code']}. description: #{transfer_data['error_desc']}"
           end
-          sleep(1)
+          sleep(1.0)
         end
         # TODO: get status of sessions
         return []

data/lib/aspera/fasp/agent_trsdk.rb

@@ -1,61 +1,121 @@
 # frozen_string_literal: true
 
-# cspell:words Transfersdk
-
 require 'aspera/fasp/agent_base'
 require 'aspera/fasp/installation'
+require 'aspera/temp_file_manager'
+require 'aspera/log'
+require 'aspera/assert'
 require 'json'
+require 'uri'
 
 module Aspera
   module Fasp
     class AgentTrsdk < Aspera::Fasp::AgentBase
+      # see https://github.com/grpc/grpc/blob/master/doc/naming.md
+      # https://grpc.io/docs/guides/custom-name-resolution/
+      LOCAL_SOCKET_ADDR = '127.0.0.1'
+      PORT_SEP = ':'
+      # port zero means select a random available high port
+      AUTO_LOCAL_TCP_PORT = "#{PORT_SEP}0"
       DEFAULT_OPTIONS = {
-        address: '127.0.0.1',
-        port:    55_002
+        url:      AUTO_LOCAL_TCP_PORT,
+        external: false,         # expect that an external daemon is already running
+        keep:     false          # do not shutdown daemon on exit
       }.freeze
       private_constant :DEFAULT_OPTIONS
 
-      # options come from transfer_info
-      def initialize(user_opts)
-        raise "expecting Hash (or nil), but have #{user_opts.class}" unless user_opts.nil? || user_opts.is_a?(Hash)
-        # set default options and override if specified
-        options = DEFAULT_OPTIONS.dup
-        user_opts&.each do |k, v|
-          raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map(&:to_s).join(',')}" unless DEFAULT_OPTIONS.key?(k)
-          options[k] = v
+      class << self
+        # Well, the port number is only in log file
+        def daemon_port_from_log(log_file)
+          result = nil
+          # if port is zero, a dynamic port was created, get it
+          File.open(log_file, 'r') do |file|
+            file.each_line do |line|
+              # Well, it's tricky to depend on log
+              if (m = line.match(/Info: API Server: Listening on ([^:]+):(\d+) /))
+                result = m[2].to_i
+                # no "break" , need to read last matching log line
+              end
+            end
+          end
+          raise 'Port not found in daemon logs' if result.nil?
+          Log.log.debug{"Got port #{result} from log"}
+          return result
         end
-        Log.log.debug{"options= #{options}"}
-        super()
-        # load and create SDK stub
+      end
+
+      # options come from transfer_info
+      def initialize(user_opts={})
+        super(user_opts)
+        @options = AgentBase.options(default: DEFAULT_OPTIONS, options: user_opts)
+        is_local_auto_port = @options[:url].eql?(AUTO_LOCAL_TCP_PORT)
+        raise 'Cannot use options `keep` or `external` with port zero' if is_local_auto_port && (@options[:keep] || @options[:external])
+        Log.log.debug{Log.dump(:agent_options, @options)}
+        # load SDK stub class on demand, as it's an optional gem
         $LOAD_PATH.unshift(Installation.instance.sdk_ruby_folder)
         require 'transfer_services_pb'
-        @transfer_client = Transfersdk::TransferService::Stub.new("#{options[:address]}:#{options[:port]}", :this_channel_is_insecure)
+        # keep PID for optional shutdown
+        @daemon_pid = nil
+        daemon_endpoint = @options[:url]
+        Log.log.debug{Log.dump(:daemon_endpoint, daemon_endpoint)}
+        # retry loop
         begin
+          # no address: local bind
+          daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{daemon_endpoint}" if daemon_endpoint.match?(/^#{PORT_SEP}[0-9]+$/o)
+          # Create stub (without credentials)
+          @transfer_client = Transfersdk::TransferService::Stub.new(daemon_endpoint, :this_channel_is_insecure)
+          # Initiate actual connection
           get_info_response = @transfer_client.get_info(Transfersdk::InstanceInfoRequest.new)
-          Log.log.debug{"daemon info: #{get_info_response}"}
-        rescue GRPC::Unavailable
-          Log.log.warn('no daemon present, starting daemon...')
+          Log.log.debug{"Daemon info: #{get_info_response}"}
+          Log.log.warn{'Attached to existing daemon'} unless @daemon_pid || @options[:external] || @options[:keep]
+          at_exit{shutdown}
+        rescue GRPC::Unavailable => e
+          # if transferd is external: do not start it, or other error
+          raise if @options[:external] || !e.message.include?('failed to connect')
+          # we already tried to start a daemon, but it failed
+          assert(@daemon_pid.nil?){"Daemon started with PID #{@daemon_pid}, but connection failed to #{daemon_endpoint}}"}
+          Log.log.warn('no daemon present, starting daemon...') if @options[:external]
           # location of daemon binary
-          bin_folder = File.realpath(File.join(Installation.instance.sdk_ruby_folder, '..'))
-          # config file and logs are created in same folder
-          conf_file = File.join(bin_folder, 'sdk.conf')
-          log_base = File.join(bin_folder, 'transferd')
+          sdk_folder = File.realpath(File.join(Installation.instance.sdk_ruby_folder, '..'))
+          # transferd only supports local ip and port
+          daemon_uri = URI.parse("ipv4://#{daemon_endpoint}")
+          assert(daemon_uri.scheme.eql?('ipv4')){"Invalid scheme daemon URI #{daemon_endpoint}"}
           # create a config file for daemon
           config = {
-            address:      options[:address],
-            port:         options[:port],
+            address:      daemon_uri.host,
+            port:         daemon_uri.port,
             fasp_runtime: {
               use_embedded: false,
               user_defined: {
-                bin: bin_folder,
-                etc: bin_folder
+                bin: sdk_folder,
+                etc: sdk_folder
               }
             }
           }
+          # config file and logs are created in same folder
+          transferd_base_tmp = TempFileManager.instance.new_file_path_global('transferd')
+          Log.log.debug{"transferd base tmp #{transferd_base_tmp}"}
+          conf_file = "#{transferd_base_tmp}.conf"
+          log_stdout = "#{transferd_base_tmp}.out"
+          log_stderr = "#{transferd_base_tmp}.err"
           File.write(conf_file, config.to_json)
-          trd_pid = Process.spawn(Installation.instance.path(:transferd), '--config', conf_file, out: "#{log_base}.out", err: "#{log_base}.err")
-          Process.detach(trd_pid)
-          sleep(2.0)
+          @daemon_pid = Process.spawn(Installation.instance.path(:transferd), '--config', conf_file, out: log_stdout, err: log_stderr)
+          begin
+            # wait for process to initialize, max 2 seconds
+            Timeout.timeout(2.0) do
+              # this returns if process dies (within 2 seconds)
+              _, status = Process.wait2(@daemon_pid)
+              raise "Transfer daemon exited with status #{status.exitstatus}. Check files: #{log_stdout} and #{log_stderr}"
+            end
+          rescue Timeout::Error
+            nil
+          end
+          Log.log.debug{"Daemon started with pid #{@daemon_pid}"}
+          Process.detach(@daemon_pid) if @options[:keep]
+          at_exit {shutdown}
+          # update port for next connection attempt (if auto high port was requested)
+          daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{PORT_SEP}#{self.class.daemon_port_from_log(log_stdout)}" if is_local_auto_port
+          # local daemon started, try again
           retry
         end
       end
@@ -74,25 +134,34 @@ module Aspera
       end
 
       def wait_for_transfers_completion
-        started = false
+        # set to true when we know the total size of the transfer
+        session_started = false
+        bytes_expected = nil
         # monitor transfer status
         @transfer_client.monitor_transfers(Transfersdk::RegistrationRequest.new(transferId: [@transfer_id])) do |response|
-          Log.dump(:response, response.to_h)
+          Log.log.debug{Log.dump(:response, response.to_h)}
           # Log.log.debug{"#{response.sessionInfo.preTransferBytes} #{response.transferInfo.bytesTransferred}"}
           case response.status
           when :RUNNING
-            if !started && !response.sessionInfo.preTransferBytes.eql?(0)
-              notify_begin(@transfer_id, response.sessionInfo.preTransferBytes)
-              started = true
-            elsif started
-              notify_progress(@transfer_id, response.transferInfo.bytesTransferred)
+            if !session_started
+              notify_progress(session_id: @transfer_id, type: :session_start)
+              session_started = true
             end
-          when :FAILED, :COMPLETED, :CANCELED
-            notify_end(@transfer_id)
-            raise Fasp::Error, JSON.parse(response.message)['Description'] unless :COMPLETED.eql?(response.status)
+            if bytes_expected.nil? &&
+                !response.sessionInfo.preTransferBytes.eql?(0)
+              bytes_expected = response.sessionInfo.preTransferBytes
+              notify_progress(type: :session_size, session_id: @transfer_id, info: bytes_expected)
+            end
+            notify_progress(type: :transfer, session_id: @transfer_id, info: response.transferInfo.bytesTransferred)
+          when :COMPLETED
+            notify_progress(type: :transfer, session_id: @transfer_id, info: bytes_expected) if bytes_expected
+            notify_progress(type: :end, session_id: @transfer_id)
             break
+          when :FAILED, :CANCELED
+            notify_progress(type: :end, session_id: @transfer_id)
+            raise Fasp::Error, JSON.parse(response.message)['Description']
           when :QUEUED, :UNKNOWN_STATUS, :PAUSED, :ORPHANED
-            # ignore
+            notify_progress(session_id: nil, type: :pre_start, info: response.status.to_s.downcase)
           else
             Log.log.error{"unknown status#{response.status}"}
           end
@@ -100,6 +169,20 @@ module Aspera
         # TODO: return status
         return []
       end
+
+      def shutdown
+        stop_daemon unless @options[:keep]
+      end
+
+      def stop_daemon
+        if !@daemon_pid.nil?
+          Log.log.debug("Stopping daemon #{@daemon_pid}")
+          Process.kill('INT', @daemon_pid)
+          _, status = Process.wait2(@daemon_pid)
+          Log.log.debug("daemon stopped #{status}")
+          @daemon_pid = nil
+        end
+      end
     end
   end
 end

data/lib/aspera/fasp/error_info.rb

@@ -5,11 +5,11 @@
 module Aspera
   module Fasp
     # from https://www.google.com/search?q=FASP+error+codes
-    # Note that the fact that an error is retryable is not internally defined by protocol, it's client-side responsibility
+    # Note that the fact that an error is retry-able is not internally defined by protocol, it's client-side responsibility
     # rubocop:disable Layout/MultilineHashKeyLineBreaks
     # rubocop:disable Layout/FirstHashElementLineBreak
     ERROR_INFO = {
-      # id   retryable      mnemo                       message                                              additional info
+      # id   retry-able     mnemo                       message                                              additional info
       1  => { r: false, c: 'FASP_PROTO',                  m: 'Generic fasp(tm) protocol error',                a: 'fasp(tm) error'},
       2  => { r: false, c: 'ASCP',                        m: 'Generic SCP error',                              a: 'ASCP error'},
       3  => { r: false, c: 'AMBIGUOUS_TARGET',            m: 'Target incorrectly specified',                   a: 'Ambiguous target'},

data/lib/aspera/fasp/faux_file.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Aspera
+  module Fasp
+    # generates a pseudo file stream
+    class FauxFile
+      # marker for faux file
+      PREFIX = 'faux:///'
+      # size suffix
+      SUFFIX = %w[k m g t p e]
+      class << self
+        def open(name)
+          return nil unless name.start_with?(PREFIX)
+          parts = name[PREFIX.length..-1].split('?')
+          raise 'Format: #{PREFIX}<file path>?<size>' unless parts.length.eql?(2)
+          raise "Format: <integer>[#{SUFFIX.join(',')}]" unless (m = parts[1].downcase.match(/^(\d+)([#{SUFFIX.join('')}])$/))
+          size = m[1].to_i
+          suffix = m[2]
+          SUFFIX.each do |s|
+            size *= 1024
+            break if s.eql?(suffix)
+          end
+          return FauxFile.new(parts[0], size)
+        end
+      end
+      attr_reader :path, :size
+
+      def initialize(path, size)
+        @path = path
+        @size = size
+        @offset = 0
+        # we cache large chunks, anyway most of them will be the same size
+        @chunk_by_size = {}
+      end
+
+      def read(chunk_size)
+        return nil if eof?
+        bytes_to_read = [chunk_size, @size - @offset].min
+        @offset += bytes_to_read
+        @chunk_by_size[bytes_to_read] = "\x00" * bytes_to_read unless @chunk_by_size.key?(bytes_to_read)
+        return @chunk_by_size[bytes_to_read]
+      end
+
+      def close
+      end
+
+      def eof?
+        return @offset >= @size
+      end
+    end
+  end
+end