aspera-cli 4.18.0 → 4.19.0

data/lib/aspera/agent/alpha.rb

@@ -4,7 +4,7 @@ require 'aspera/agent/base'
 require 'aspera/rest'
 require 'aspera/log'
 require 'aspera/json_rpc'
-require 'aspera/open_application'
+require 'aspera/environment'
 require 'securerandom'
 
 module Aspera
@@ -22,7 +22,7 @@ module Aspera
         @application_id = SecureRandom.uuid
         @xfer_id = nil
         super(**base_options)
-        raise 'Using client requires a graphical environment' if !OpenApplication.default_gui_mode.eql?(:graphical)
+        raise 'Using client requires a graphical environment' if !Environment.default_gui_mode.eql?(:graphical)
         method_index = 0
         begin
           # curl 'http://127.0.0.1:33024/' -X POST -H 'content-type: application/json' --data-raw '{"jsonrpc":"2.0","params":[],"id":999999,"method":"rpc.discover"}'
@@ -36,8 +36,8 @@ module Aspera
           method_index += 1
           raise StandardError, "Unable to start #{APP_NAME} #{method_index} times" if start_url.nil?
           Log.log.warn{"#{APP_NAME} is not started (#{e}). Trying to start it ##{method_index}..."}
-          if !OpenApplication.uri_graphical(start_url)
-            OpenApplication.uri_graphical('https://www.ibm.com/aspera/connect/')
+          if !Environment.open_uri_graphical(start_url)
+            Environment.open_uri_graphical('https://www.ibm.com/aspera/connect/')
             raise StandardError, "#{APP_NAME} is not installed"
           end
           sleep(SLEEP_SEC_BETWEEN_RETRY)

data/lib/aspera/agent/base.rb

@@ -38,6 +38,8 @@ module Aspera
           end.map{|file|file[0..(-1 - RUBY_EXT.length)].to_sym}
         end
       end
+
+      # Wait for all sessions to terminate and return the status of each session
       def wait_for_completion
         # list of: :success or "error message string"
         statuses = wait_for_transfers_completion

data/lib/aspera/agent/connect.rb

@@ -2,7 +2,6 @@
 
 require 'aspera/agent/base'
 require 'aspera/rest'
-require 'aspera/open_application'
 require 'securerandom'
 
 module Aspera
@@ -18,7 +17,7 @@ module Aspera
         @connect_settings = {
           'app_id' => SecureRandom.uuid
         }
-        raise 'Using connect requires a graphical environment' if !OpenApplication.default_gui_mode.eql?(:graphical)
+        raise 'Using connect requires a graphical environment' if !Environment.default_gui_mode.eql?(:graphical)
         method_index = 0
         begin
           connect_url = Ascp::Products.connect_uri
@@ -34,8 +33,8 @@ module Aspera
           method_index += 1
           raise StandardError, "Unable to start connect #{method_index} times" if start_url.nil?
           Log.log.warn{"Aspera Connect is not started (#{e}). Trying to start it ##{method_index}..."}
-          if !OpenApplication.uri_graphical(start_url)
-            OpenApplication.uri_graphical('https://www.ibm.com/aspera/connect/')
+          if !Environment.open_uri_graphical(start_url)
+            Environment.open_uri_graphical('https://www.ibm.com/aspera/connect/')
             raise StandardError, 'Connect is not installed'
           end
           sleep(SLEEP_SEC_BETWEEN_RETRY)

data/lib/aspera/agent/direct.rb

@@ -19,9 +19,10 @@ module Aspera
     # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
     class Direct < Base
       LISTEN_LOCAL_ADDRESS = '127.0.0.1'
-      ANY_AVAILABLE_PORT = 0 # 0 means any available port
+      # 0 means: select an available port
+      SELECT_AVAILABLE_PORT = 0
       # spellchecker: enable
-      private_constant :LISTEN_LOCAL_ADDRESS, :ANY_AVAILABLE_PORT
+      private_constant :LISTEN_LOCAL_ADDRESS, :SELECT_AVAILABLE_PORT
 
       # method of Base
       # start ascp transfer(s) (non blocking), single or multi-session
@@ -93,12 +94,12 @@ module Aspera
             # do deep copy (each thread has its own copy because it is modified here below and in thread)
             this_session = session.clone
             this_session[:ts] = this_session[:ts].clone
-            this_session[:env_args] = this_session[:env_args].clone
-            this_session[:env_args][:args] = this_session[:env_args][:args].clone
+            env_args = this_session[:env_args] = this_session[:env_args].clone
+            args = env_args[:args] = env_args[:args].clone
             # set multi session part
-            this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_info[:count]}")
+            args.unshift("-C#{i}:#{multi_session_info[:count]}")
             # option: increment (default as per ascp manual) or not (cluster on other side ?)
-            this_session[:env_args][:args].unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @multi_incr_udp
+            args.unshift('-O', (multi_session_info[:udp_base] + i - 1).to_s) if @multi_incr_udp
             # finally start the thread
             this_session[:thread] = Thread.new(this_session) {|session_info|transfer_thread_entry(session_info)}
             @sessions.push(this_session)
@@ -129,81 +130,46 @@ module Aspera
         Log.log.debug('fasp local shutdown')
       end
 
-      # @param event management port event
-      def process_progress(event)
-        session_id = event['SessionId']
-        case event['Type']
-        when 'INIT'
-          @pre_calc_sent = false
-          @pre_calc_last_size = nil
-          notify_progress(session_id: session_id, type: :session_start)
-        when 'NOTIFICATION' # sent from remote
-          if event.key?('PreTransferBytes')
-            @pre_calc_sent = true
-            notify_progress(session_id: session_id, type: :session_size, info: event['PreTransferBytes'])
-          end
-        when 'STATS' # during transfer
-          @pre_calc_last_size = event['TransferBytes'].to_i + event['StartByte'].to_i
-          notify_progress(session_id: session_id, type: :transfer, info: @pre_calc_last_size)
-        when 'DONE', 'ERROR' # end of session
-          total_size = event['TransferBytes'].to_i + event['StartByte'].to_i
-          if !@pre_calc_sent && !total_size.zero?
-            notify_progress(session_id: session_id, type: :session_size, info: total_size)
-          end
-          if @pre_calc_last_size != total_size
-            notify_progress(session_id: session_id, type: :transfer, info: total_size)
-          end
-          notify_progress(session_id: session_id, type: :end)
-          # cspell:disable
-        when 'SESSION'
-        when 'ARGSTOP'
-        when 'FILEERROR'
-        when 'STOP'
-          # cspell:enable
-          # stop event when one file is completed
-        else
-          Log.log.debug{"unknown event type #{event['Type']}"}
-        end
+      # @return [Array] list of sessions for a job
+      def sessions_by_job(job_id)
+        @sessions.select{|session_info| session_info[:job_id].eql?(job_id)}
       end
 
-      # This is the low level method to start the "ascp" process
-      # currently, relies on command line arguments
-      # start ascp with management port.
+      # This is the low level method to start the transfer process
+      # Start process with management port.
+      # returns
       # raises FaspError on error
-      # if there is a thread info: set and broadcast session id
-      # runs in separate thread
-      # @param env_args a hash containing :args :env :ascp_version
-      # @param session this session information
-      # could be private method
-      def start_transfer_with_args_env(env_args, session)
-        Aspera.assert_type(env_args, Hash)
+      # @param session this session information, keys :io and :token_regenerator
+      # @param env  [Hash]   environment variables
+      # @param name [Symbol] name of executable: :ascp, :ascp4 or :async
+      # @param args [Array]  command line arguments
+      # @return [nil] when process has exited
+      def start_and_monitor_process(
+        session:,
+        env:,
+        name:,
+        args:
+      )
         Aspera.assert_type(session, Hash)
-        Log.log.debug{"env_args=#{env_args.inspect}"}
         notify_progress(session_id: nil, type: :pre_start, info: 'starting')
         begin
-          ascp_pid = nil
+          command_pid = nil
           # we use Socket directly, instead of TCPServer, as it gives access to lower level options
-          socket_class = RUBY_ENGINE.eql?('jruby') ? ServerSocket : Socket
+          socket_class = defined?(JRUBY_VERSION) ? ServerSocket : Socket
           mgt_server_socket = socket_class.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
-          # open any available (0) local TCP port for use as ascp management port
-          mgt_server_socket.bind(Addrinfo.tcp(LISTEN_LOCAL_ADDRESS, ANY_AVAILABLE_PORT))
+          # open any available (0) local TCP port for use as management port
+          mgt_server_socket.bind(Addrinfo.tcp(LISTEN_LOCAL_ADDRESS, SELECT_AVAILABLE_PORT))
           # build arguments and add mgt port
-          ascp_arguments = ['-M', mgt_server_socket.local_address.ip_port.to_s].concat(env_args[:args])
-          # get location of ascp executable
-          ascp_path = Ascp::Installation.instance.path(env_args[:ascp_version])
-          # display ascp command line
-          Log.log.debug do
-            [
-              'execute:'.red,
-              env_args[:env].map{|k, v| "#{k}=#{Shellwords.shellescape(v)}"},
-              Shellwords.shellescape(ascp_path),
-              ascp_arguments.map{|a|Shellwords.shellescape(a)}
-            ].flatten.join(' ')
+          command_arguments = if name.eql?(:async)
+            ["--exclusive-mgmt-port=#{mgt_server_socket.local_address.ip_port}"]
+          else
+            ['-M', mgt_server_socket.local_address.ip_port.to_s]
           end
-          # start ascp in separate process
-          ascp_pid = Process.spawn(env_args[:env], [ascp_path, ascp_path], *ascp_arguments, close_others: true)
-          Log.log.debug{"spawned ascp pid #{ascp_pid}"}
-          notify_progress(session_id: nil, type: :pre_start, info: 'waiting for ascp')
+          command_arguments.concat(args)
+          # get location of command executable (ascp, async)
+          command_path = Ascp::Installation.instance.path(name)
+          command_pid = Environment.secure_spawn(env: env, exec: command_path, args: command_arguments)
+          notify_progress(session_id: nil, type: :pre_start, info: "waiting for #{name} to start")
           mgt_server_socket.listen(1)
           # TODO: timeout does not work when Process.spawn is used... until process exits, then it works
           Log.log.debug{"before select, timeout: #{@spawn_timeout_sec}"}
@@ -213,15 +179,15 @@ module Aspera
           # There is a connection to accept
           client_socket, _client_addrinfo = mgt_server_socket.accept
           Log.log.debug('after accept')
-          ascp_mgt_io = client_socket.to_io
+          management_port_io = client_socket.to_io
           # management messages include file names which may be utf8
           # by default socket is US-ASCII
           # TODO: use same value as Encoding.default_external
-          ascp_mgt_io.set_encoding(Encoding::UTF_8)
-          session[:io] = ascp_mgt_io
+          management_port_io.set_encoding(Encoding::UTF_8)
+          session[:io] = management_port_io
           processor = Ascp::Management.new
           # read management port, until socket is closed (gets returns nil)
-          while (line = ascp_mgt_io.gets)
+          while (line = management_port_io.gets)
             event = processor.process_line(line.chomp)
             next unless event
             # event is ready
@@ -241,7 +207,7 @@ module Aspera
                 # regenerate token here, expired, or error on it
                 # Note: in multi-session, each session will have a different one.
                 Log.log.warn('Regenerating token for transfer')
-                env_args[:env]['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
+                env['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
               end
               raise Transfer::Error.new(last_event['Description'], last_event['Code'].to_i)
             when 'DONE'
@@ -257,16 +223,16 @@ module Aspera
           raise Transfer::Error, 'transfer interrupted by user'
         ensure
           mgt_server_socket.close
-          # if ascp was successfully started, check its status
-          unless ascp_pid.nil?
+          # if command was successfully started, check its status
+          unless command_pid.nil?
             # "wait" for process to avoid zombie
-            Process.wait(ascp_pid)
+            Process.wait(command_pid)
             status = $CHILD_STATUS
-            ascp_pid = nil
+            # command_pid = nil
             session.delete(:io)
-            # status is nil if an exception occurred before starting ascp
+            # status is nil if an exception occurred before starting command
             if !status&.success?
-              message = status.nil? ? 'ascp not started' : "ascp failed (#{status})"
+              message = status.nil? ? "#{name} not started" : "#{name} failed (#{status})"
               # raise error only if there was not already an exception (ERROR_INFO)
               raise Transfer::Error, message unless $ERROR_INFO
               # else display this message also, as main exception is already here
@@ -274,11 +240,47 @@ module Aspera
             end
           end
         end
+        nil
       end
 
-      # @return [Array] list of sessions for a job
-      def sessions_by_job(job_id)
-        @sessions.select{|session_info| session_info[:job_id].eql?(job_id)}
+      private
+
+      # notify progress to callback
+      # @param event management port event
+      def process_progress(event)
+        session_id = event['SessionId']
+        case event['Type']
+        when 'INIT'
+          @pre_calc_sent = false
+          @pre_calc_last_size = nil
+          notify_progress(session_id: session_id, type: :session_start)
+        when 'NOTIFICATION' # sent from remote
+          if event.key?('PreTransferBytes')
+            @pre_calc_sent = true
+            notify_progress(session_id: session_id, type: :session_size, info: event['PreTransferBytes'])
+          end
+        when 'STATS' # during transfer
+          @pre_calc_last_size = event['TransferBytes'].to_i + event['StartByte'].to_i
+          notify_progress(session_id: session_id, type: :transfer, info: @pre_calc_last_size)
+        when 'DONE', 'ERROR' # end of session
+          total_size = event['TransferBytes'].to_i + event['StartByte'].to_i
+          if !@pre_calc_sent && !total_size.zero?
+            notify_progress(session_id: session_id, type: :session_size, info: total_size)
+          end
+          if @pre_calc_last_size != total_size
+            notify_progress(session_id: session_id, type: :transfer, info: total_size)
+          end
+          notify_progress(session_id: session_id, type: :end)
+          # cspell:disable
+        when 'SESSION'
+        when 'ARGSTOP'
+        when 'FILEERROR'
+        when 'STOP'
+          # cspell:enable
+          # stop event when one file is completed
+        else
+          Log.log.debug{"unknown event type #{event['Type']}"}
+        end
       end
 
       # @return [Hash] session information
@@ -289,7 +291,7 @@ module Aspera
         return matches.first
       end
 
-      # send command of management port to ascp session (used in `asession)
+      # send command to management port of command (used in `asession)
       # @param job_id identified transfer process
       # @param session_index index of session (for multi session)
       # @param data command on mgt port, examples:
@@ -309,47 +311,49 @@ module Aspera
       end
       attr_reader :sessions
 
-      private
-
       # options for initialize (same as values in option transfer_info)
-      # @param wss [Boolean] true: if both SSH and wss in ts: prefer wss
       # @param ascp_args [Array] additional arguments to ascp
-      # @param spawn_timeout_sec [Integer] timeout for ascp spawn
-      # @param spawn_delay_sec [Integer] optional delay to start between sessions
-      # @param multi_incr_udp [Boolean] true: increment udp port for each session
-      # @param trusted_certs [Array] list of files with trusted certificates (stores)
-      # @param resume [Hash] resume policy
+      # @param wss [Boolean] true: if both SSH and wss in ts: prefer wss
       # @param quiet [Boolean] by default no native ascp progress bar
+      # @param trusted_certs [Array,NilClass] list of files with trusted certificates (stores)
+      # @param client_ssh_key [String] client ssh key option (from CLIENT_SSH_KEY_OPTIONS)
       # @param check_ignore_cb [Proc] callback with host,port
+      # @param spawn_timeout_sec [Integer] timeout for ascp spawn
+      # @param spawn_delay_sec [Integer] optional delay to start between sessions
+      # @param multi_incr_udp [Boolean,NilClass] true: increment udp port for each session
+      # @param resume [Hash,NilClass] resume policy
       # @param management_cb [Proc] callback for management events
       # @param base_options [Hash] other options for base class
       def initialize(
+        ascp_args:         nil,
         wss:               true,
-        ascp_args:         [],
-        spawn_timeout_sec: 2,
-        spawn_delay_sec:   2,
-        multi_incr_udp:    true,
-        trusted_certs:     [],
-        resume:            {},
         quiet:             true,
+        trusted_certs:     nil,
+        client_ssh_key:    nil,
         check_ignore_cb:   nil,
+        spawn_timeout_sec: 2,
+        spawn_delay_sec:   2,
+        multi_incr_udp:    nil,
+        resume:            nil,
         management_cb:     nil,
         **base_options
       )
         super(**base_options)
+        # special transfer parameters
         @tr_opts = {
           ascp_args:       ascp_args,
           wss:             wss,
           quiet:           quiet,
           trusted_certs:   trusted_certs,
+          client_ssh_key:  client_ssh_key,
           check_ignore_cb: check_ignore_cb
         }
         @spawn_timeout_sec = spawn_timeout_sec
         @spawn_delay_sec = spawn_delay_sec
-        @multi_incr_udp = multi_incr_udp
-        @resume = resume
+        # default is true on windows, false on other platforms
+        @multi_incr_udp = multi_incr_udp.nil? ? Environment.os.eql?(Environment::OS_WINDOWS) : multi_incr_udp
         @management_cb = management_cb
-        @resume_policy = Resumer.new(@resume.symbolize_keys)
+        @resume_policy = Resumer.new(resume.nil? ? {} : resume.symbolize_keys)
         # all transfer jobs, key = SecureRandom.uuid, protected by mutex, cond var on change
         @sessions = []
         # mutex protects global data accessed by threads
@@ -365,7 +369,7 @@ module Aspera
           Log.log.debug{"ENTER (#{Thread.current[:name]})"}
           # start transfer with selected resumer policy
           @resume_policy.execute_with_resume do
-            start_transfer_with_args_env(session[:env_args], session)
+            start_and_monitor_process(session: session, **session[:env_args])
           end
           Log.log.debug('transfer ok'.bg_green)
         rescue StandardError => e

data/lib/aspera/agent/httpgw.rb

@@ -50,7 +50,7 @@ module Aspera
         super(**base_options)
         @gw_api = Api::Httpgw.new(
           # remove /v1 from end of user-provided GW url: we need the base url only
-          url:               url.gsub(%r{/#{Api::Httpgw::API_V1}/*$}o, ''),
+          url:               url,
           api_version:       api_version,
           upload_chunk_size: upload_chunk_size,
           synchronous:       synchronous,

data/lib/aspera/api/aoc.rb

@@ -439,7 +439,7 @@ module Aspera
       # create a package
       # @param package_data [Hash] package creation (with extensions...)
       # @param validate_meta [TrueClass,FalseClass] true to validate parameters locally
-      # @param new_user_option [Hash] options if an unknown user is specified
+      # @param new_user_option [Hash,NilClass] options if an unknown user is specified
       # @return transfer spec, node api and package information
       def create_package_simple(package_data, validate_meta, new_user_option)
         update_package_metadata_for_api(package_data)
@@ -476,7 +476,7 @@ module Aspera
       # callback in Node (transfer_spec_gen4)
       def add_ts_tags(transfer_spec:, app_info:)
         # translate transfer direction to upload/download
-        transfer_type = Transfer::Spec.action(transfer_spec)
+        transfer_type = Transfer::Spec.direction_to_transfer_type(transfer_spec['direction'])
         # Analytics tags
         ################
         transfer_spec.deep_merge!({