aspera-cli 4.15.0 → 4.17.0

data/lib/aspera/agent/trsdk.rb

@@ -0,0 +1,188 @@
+# frozen_string_literal: true
+
+require 'aspera/agent/base'
+require 'aspera/ascp/installation'
+require 'aspera/temp_file_manager'
+require 'aspera/log'
+require 'aspera/assert'
+require 'json'
+require 'uri'
+
+module Aspera
+  module Agent
+    class Trsdk < Base
+      # see https://github.com/grpc/grpc/blob/master/doc/naming.md
+      # https://grpc.io/docs/guides/custom-name-resolution/
+      LOCAL_SOCKET_ADDR = '127.0.0.1'
+      PORT_SEP = ':'
+      # port zero means select a random available high port
+      AUTO_LOCAL_TCP_PORT = "#{PORT_SEP}0"
+      DEFAULT_OPTIONS = {
+        url:      AUTO_LOCAL_TCP_PORT,
+        external: false,         # expect that an external daemon is already running
+        keep:     false          # do not shutdown daemon on exit
+      }.freeze
+      private_constant :DEFAULT_OPTIONS
+
+      class << self
+        # Well, the port number is only in log file
+        def daemon_port_from_log(log_file)
+          result = nil
+          # if port is zero, a dynamic port was created, get it
+          File.open(log_file, 'r') do |file|
+            file.each_line do |line|
+              # Well, it's tricky to depend on log
+              if (m = line.match(/Info: API Server: Listening on ([^:]+):(\d+) /))
+                result = m[2].to_i
+                # no "break" , need to read last matching log line
+              end
+            end
+          end
+          raise 'Port not found in daemon logs' if result.nil?
+          Log.log.debug{"Got port #{result} from log"}
+          return result
+        end
+      end
+
+      # options come from transfer_info
+      def initialize(user_opts={})
+        super(user_opts)
+        @options = Base.options(default: DEFAULT_OPTIONS, options: user_opts)
+        is_local_auto_port = @options[:url].eql?(AUTO_LOCAL_TCP_PORT)
+        raise 'Cannot use options `keep` or `external` with port zero' if is_local_auto_port && (@options[:keep] || @options[:external])
+        Log.log.debug{Log.dump(:agent_options, @options)}
+        # load SDK stub class on demand, as it's an optional gem
+        $LOAD_PATH.unshift(Ascp::Installation.instance.sdk_ruby_folder)
+        require 'transfer_services_pb'
+        # keep PID for optional shutdown
+        @daemon_pid = nil
+        daemon_endpoint = @options[:url]
+        Log.log.debug{Log.dump(:daemon_endpoint, daemon_endpoint)}
+        # retry loop
+        begin
+          # no address: local bind
+          daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{daemon_endpoint}" if daemon_endpoint.match?(/^#{PORT_SEP}[0-9]+$/o)
+          # Create stub (without credentials)
+          @transfer_client = Transfersdk::TransferService::Stub.new(daemon_endpoint, :this_channel_is_insecure)
+          # Initiate actual connection
+          get_info_response = @transfer_client.get_info(Transfersdk::InstanceInfoRequest.new)
+          Log.log.debug{"Daemon info: #{get_info_response}"}
+          Log.log.warn{'Attached to existing daemon'} unless @daemon_pid || @options[:external] || @options[:keep]
+          at_exit{shutdown}
+        rescue GRPC::Unavailable => e
+          # if transferd is external: do not start it, or other error
+          raise if @options[:external] || !e.message.include?('failed to connect')
+          # we already tried to start a daemon, but it failed
+          Aspera.assert(@daemon_pid.nil?){"Daemon started with PID #{@daemon_pid}, but connection failed to #{daemon_endpoint}}"}
+          Log.log.warn('no daemon present, starting daemon...') if @options[:external]
+          # location of daemon binary
+          sdk_folder = File.realpath(File.join(Ascp::Installation.instance.sdk_ruby_folder, '..'))
+          # transferd only supports local ip and port
+          daemon_uri = URI.parse("ipv4://#{daemon_endpoint}")
+          Aspera.assert(daemon_uri.scheme.eql?('ipv4')){"Invalid scheme daemon URI #{daemon_endpoint}"}
+          # create a config file for daemon
+          config = {
+            address:      daemon_uri.host,
+            port:         daemon_uri.port,
+            fasp_runtime: {
+              use_embedded: false,
+              user_defined: {
+                bin: sdk_folder,
+                etc: sdk_folder
+              }
+            }
+          }
+          # config file and logs are created in same folder
+          transferd_base_tmp = TempFileManager.instance.new_file_path_global('transferd')
+          Log.log.debug{"transferd base tmp #{transferd_base_tmp}"}
+          conf_file = "#{transferd_base_tmp}.conf"
+          log_stdout = "#{transferd_base_tmp}.out"
+          log_stderr = "#{transferd_base_tmp}.err"
+          File.write(conf_file, config.to_json)
+          @daemon_pid = Process.spawn(Ascp::Installation.instance.path(:transferd), '--config', conf_file, out: log_stdout, err: log_stderr)
+          begin
+            # wait for process to initialize, max 2 seconds
+            Timeout.timeout(2.0) do
+              # this returns if process dies (within 2 seconds)
+              _, status = Process.wait2(@daemon_pid)
+              raise "Transfer daemon exited with status #{status.exitstatus}. Check files: #{log_stdout} and #{log_stderr}"
+            end
+          rescue Timeout::Error
+            nil
+          end
+          Log.log.debug{"Daemon started with pid #{@daemon_pid}"}
+          Process.detach(@daemon_pid) if @options[:keep]
+          at_exit {shutdown}
+          # update port for next connection attempt (if auto high port was requested)
+          daemon_endpoint = "#{LOCAL_SOCKET_ADDR}#{PORT_SEP}#{self.class.daemon_port_from_log(log_stdout)}" if is_local_auto_port
+          # local daemon started, try again
+          retry
+        end
+      end
+
+      def start_transfer(transfer_spec, token_regenerator: nil)
+        # create a transfer request
+        transfer_request = Transfersdk::TransferRequest.new(
+          transferType: Transfersdk::TransferType::FILE_REGULAR, # transfer type (file/stream)
+          config: Transfersdk::TransferConfig.new, # transfer configuration
+          transferSpec: transfer_spec.to_json) # transfer definition
+        # send start transfer request to the transfer manager daemon
+        start_transfer_response = @transfer_client.start_transfer(transfer_request)
+        Log.log.debug{"start transfer response #{start_transfer_response}"}
+        @transfer_id = start_transfer_response.transferId
+        Log.log.debug{"transfer started with id #{@transfer_id}"}
+      end
+
+      def wait_for_transfers_completion
+        # set to true when we know the total size of the transfer
+        session_started = false
+        bytes_expected = nil
+        # monitor transfer status
+        @transfer_client.monitor_transfers(Transfersdk::RegistrationRequest.new(transferId: [@transfer_id])) do |response|
+          Log.log.debug{Log.dump(:response, response.to_h)}
+          # Log.log.debug{"#{response.sessionInfo.preTransferBytes} #{response.transferInfo.bytesTransferred}"}
+          case response.status
+          when :RUNNING
+            if !session_started
+              notify_progress(session_id: @transfer_id, type: :session_start)
+              session_started = true
+            end
+            if bytes_expected.nil? &&
+                !response.sessionInfo.preTransferBytes.eql?(0)
+              bytes_expected = response.sessionInfo.preTransferBytes
+              notify_progress(type: :session_size, session_id: @transfer_id, info: bytes_expected)
+            end
+            notify_progress(type: :transfer, session_id: @transfer_id, info: response.transferInfo.bytesTransferred)
+          when :COMPLETED
+            notify_progress(type: :transfer, session_id: @transfer_id, info: bytes_expected) if bytes_expected
+            notify_progress(type: :end, session_id: @transfer_id)
+            break
+          when :FAILED, :CANCELED
+            notify_progress(type: :end, session_id: @transfer_id)
+            raise Transfer::Error, JSON.parse(response.message)['Description']
+          when :QUEUED, :UNKNOWN_STATUS, :PAUSED, :ORPHANED
+            notify_progress(session_id: nil, type: :pre_start, info: response.status.to_s.downcase)
+          else
+            Log.log.error{"unknown status#{response.status}"}
+          end
+        end
+        # TODO: return status
+        return []
+      end
+
+      def shutdown
+        stop_daemon unless @options[:keep]
+      end
+
+      def stop_daemon
+        if !@daemon_pid.nil?
+          Log.log.debug("Stopping daemon #{@daemon_pid}")
+          Process.kill('INT', @daemon_pid)
+          _, status = Process.wait2(@daemon_pid)
+          Log.log.debug("daemon stopped #{status}")
+          @daemon_pid = nil
+        end
+      end
+    end
+  end
+end