aspera-cli 4.2.1 → 4.5.0

data/lib/aspera/command_line_builder.rb

@@ -4,21 +4,45 @@ module Aspera
   # process_param is called repeatedly with all known parameters
   # add_env_args is called to get resulting param list and env var (also checks that all params were used)
   class CommandLineBuilder
+    # transform yes/no to trye/false
+    def self.yes_to_true(value)
+      case value
+      when 'yes'; return true
+      when 'no'; return false
+      end
+      raise "unsupported value: #{value}"
+    end
 
-    private
-    # default value for command line based on option name
-    def switch_name(param_name,options)
-      return options[:option_switch] if options.has_key?(:option_switch)
-      return '--'+param_name.to_s.gsub('_','-')
+    # Called by provider of definition before constructor of this class so that params_definition has all mandatory fields
+    def self.normalize_description(d)
+      d.each do |param_name,options|
+        raise "Expecting Hash, but have #{options.class} in #{param_name}" unless options.is_a?(Hash)
+        #options[:accepted_types]=:bool if options[:cltype].eql?(:envvar) and !options.has_key?(:accepted_types)
+        # by default : not mandatory
+        options[:mandatory]||=false
+        options[:desc]||=''
+        # by default : string, unless it's without arg
+        if ! options.has_key?(:accepted_types)
+          options[:accepted_types]=options[:cltype].eql?(:opt_without_arg) ? :bool : :string
+        end
+        # single type is placed in array
+        options[:accepted_types]=[options[:accepted_types]] unless options[:accepted_types].is_a?(Array)
+        if !options.has_key?(:option_switch) and options.has_key?(:cltype) and [:opt_without_arg,:opt_with_arg].include?(options[:cltype])
+          options[:option_switch]='--'+param_name.to_s.gsub('_','-')
+        end
+      end
     end
 
+    private
+
+    # clvarname : command line variable name
     def env_name(param_name,options)
-      return options[:variable]
+      return options[:clvarname]
     end
 
     public
 
-    BOOLEAN_CLASSES=[TrueClass,FalseClass]
+    attr_reader :params_definition
 
     # @param param_hash
     def initialize(param_hash,params_definition)
@@ -44,15 +68,6 @@ module Aspera
       return nil
     end
 
-    # transform yes/no to trye/false
-    def self.yes_to_true(value)
-      case value
-      when 'yes'; return true
-      when 'no'; return false
-      end
-      raise "unsupported value: #{value}"
-    end
-
     # add options directly to ascp command line
     def add_command_line_options(options)
       return if options.nil?
@@ -71,24 +86,25 @@ module Aspera
     # @param options : options for type
     def process_param(param_name,action=nil)
       options=@params_definition[param_name]
-      action=options[:type] if action.nil?
+      action=options[:cltype] if action.nil?
       # should not happen
       raise "Internal error: ask processing of param #{param_name}" if options.nil?
-      # by default : not mandatory
-      options[:mandatory]||=false
-      if options.has_key?(:accepted_types)
-        # single type is placed in array
-        options[:accepted_types]=[options[:accepted_types]] unless options[:accepted_types].is_a?(Array)
-      else
-        # by default : string, unless it's without arg
-        options[:accepted_types]=action.eql?(:opt_without_arg) ? BOOLEAN_CLASSES : [String]
-      end
       # check mandatory parameter (nil is valid value)
       raise Fasp::Error.new("mandatory parameter: #{param_name}") if options[:mandatory] and !@param_hash.has_key?(param_name)
       parameter_value=@param_hash[param_name]
-      parameter_value=options[:default] if parameter_value.nil? and options.has_key?(:default)
+      #parameter_value=options[:default] if parameter_value.nil? and options.has_key?(:default)
+      expected_classes=options[:accepted_types].map do |s|
+        case s
+        when :string; String
+        when :array; Array
+        when :hash; Hash
+        when :int; Integer
+        when :bool; [TrueClass,FalseClass]
+        else raise "INTERNAL: unexpected value: #{s}"
+        end
+      end.flatten
       # check provided type
-      raise Fasp::Error.new("#{param_name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, ") unless parameter_value.nil? or options[:accepted_types].inject(false){|m,v|m or parameter_value.is_a?(v)}
+      raise Fasp::Error.new("#{param_name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, ") unless parameter_value.nil? or expected_classes.include?(parameter_value.class)
       @used_param_names.push(param_name) unless action.eql?(:defer)
 
       # process only non-nil values
@@ -102,7 +118,8 @@ module Aspera
       end
       raise "unsupported value: #{parameter_value}" unless options[:accepted_values].nil? or options[:accepted_values].include?(parameter_value)
       if options[:encode]
-        newvalue=options[:encode].call(parameter_value)
+        # :encode has name of class with encoding method
+        newvalue=Kernel.const_get(options[:encode]).send("encode_#{param_name}",parameter_value)
         raise Fasp::Error.new("unsupported #{param_name}: #{parameter_value}") if newvalue.nil?
         parameter_value=newvalue
       end
@@ -123,12 +140,12 @@ module Aspera
         else raise Fasp::Error.new("unsupported #{param_name}: #{parameter_value}")
         end
         add_param=!add_param if options[:add_on_false]
-        add_command_line_options([switch_name(param_name,options)]) if add_param
+        add_command_line_options([options[:option_switch]]) if add_param
       when :opt_with_arg # transform into command line option with value
         #parameter_value=parameter_value.to_s if parameter_value.is_a?(Integer)
         parameter_value=[parameter_value] unless parameter_value.is_a?(Array)
         # if transfer_spec value is an array, applies option many times
-        parameter_value.each{|v|add_command_line_options([switch_name(param_name,options),v])}
+        parameter_value.each{|v|add_command_line_options([options[:option_switch],v])}
       else
         raise "Error"
       end

data/lib/aspera/cos_node.rb

@@ -6,6 +6,7 @@ module Aspera
   class CosNode < Rest
     attr_reader :add_ts
     IBM_CLOUD_TOKEN_URL='https://iam.cloud.ibm.com/identity'
+    TOKEN_FIELD='delegated_refresh_token'
     def initialize(bucket_name,storage_endpoint,instance_id,api_key,auth_url=IBM_CLOUD_TOKEN_URL)
       @auth_url=auth_url
       @api_key=api_key
@@ -32,7 +33,7 @@ module Aspera
       # prepare transfer spec addition
       @add_ts={'tags'=>{'aspera'=>{'node'=>{'storage_credentials'=>{
         'type'  => 'token',
-        'token' => {'delegated_refresh_token'=>nil}
+        'token' => {TOKEN_FIELD=>nil}
         }}}}}
       generate_token
     end
@@ -45,10 +46,10 @@ module Aspera
         :base_url   => @auth_url,
         :grant      => :delegated_refresh,
         :api_key    => @api_key,
-        :token_field=> 'delegated_refresh_token'
+        :token_field=> TOKEN_FIELD
       })
       # get delagated token to be placed in rest call header and in transfer tags
-      @add_ts['tags']['aspera']['node']['storage_credentials']['token']['delegated_refresh_token']=delegated_oauth.get_authorization().gsub(/^Bearer /,'')
+      @add_ts['tags']['aspera']['node']['storage_credentials']['token'][TOKEN_FIELD]=delegated_oauth.get_authorization().gsub(/^Bearer /,'')
       @params[:headers]={'X-Aspera-Storage-Credentials'=>JSON.generate(@add_ts['tags']['aspera']['node']['storage_credentials'])}
     end
   end

data/lib/aspera/environment.rb

@@ -33,12 +33,12 @@ module Aspera
     def self.cpu
       case RbConfig::CONFIG['host_cpu']
       when /x86_64/,/x64/
-        return :x86_64
+        return CPU_X86_64
       when /powerpc/
-        return :ppc64le if os.eql?(OS_LINUX)
-        return :ppc64
+        return CPU_PPC64LE if os.eql?(OS_LINUX)
+        return CPU_PPC64
       when /s390/
-        return :s390
+        return CPU_S390
       else # other
         raise "Unknown CPU: #{RbConfig::CONFIG['host_cpu']}"
       end

data/lib/aspera/fasp/{manager.rb → agent_base.rb}

@@ -2,7 +2,7 @@ module Aspera
   module Fasp
     # Base class for FASP transfer agents
     # sub classes shall implement start_transfer and shutdown
-    class Manager
+    class AgentBase
 
       private
 
@@ -68,17 +68,18 @@ module Aspera
         self
       end
 
-      # the following methods must be implemented by subclass:
-      # start_transfer(transfer_spec,options) : start and wait for completion
-      # wait_for_transfers_completion : wait for termination of all transfers, @return list of : :success or error message
-      # optional: shutdown
-      
       # This checks the validity of the value returned by wait_for_transfers_completion
       # it must be a list of :success or exception
       def self.validate_status_list(statuses)
         raise "internal error: bad statuses type: #{statuses.class}" unless statuses.is_a?(Array)
         raise "internal error: bad statuses content: #{statuses}" unless statuses.select{|i|!i.eql?(:success) and !i.is_a?(StandardError)}.empty?
       end
+
+      # the following methods must be implemented by subclass:
+      # start_transfer(transfer_spec,options) : start and wait for completion
+      # wait_for_transfers_completion : wait for termination of all transfers, @return list of : :success or error message
+      # optional: shutdown
+
     end
   end
 end

data/lib/aspera/fasp/{connect.rb → agent_connect.rb}

@@ -1,4 +1,4 @@
-require 'aspera/fasp/manager'
+require 'aspera/fasp/agent_base'
 require 'aspera/rest'
 require 'aspera/open_application'
 require 'securerandom'
@@ -6,19 +6,15 @@ require 'tty-spinner'
 
 module Aspera
   module Fasp
-    class Connect < Manager
+    class AgentConnect < AgentBase
       MAX_CONNECT_START_RETRY=3
       SLEEP_SEC_BETWEEN_RETRY=2
       private_constant :MAX_CONNECT_START_RETRY,:SLEEP_SEC_BETWEEN_RETRY
-      def initialize
-        super
+      def initialize(options)
+        super()
         @connect_settings={
           'app_id' => SecureRandom.uuid
         }
-        # TODO: start here and create monitor
-      end
-
-      def start_transfer(transfer_spec,options=nil)
         raise 'Using connect requires a graphical environment' if !OpenApplication.default_gui_mode.eql?(:graphical)
         trynumber=0
         begin
@@ -26,6 +22,7 @@ module Aspera
           Log.log.debug("found: #{connect_url}")
           @connect_api=Rest.new({base_url: "#{connect_url}/v5/connect",headers: {'Origin'=>Rest.user_agent}}) # could use v6 also now
           cinfo=@connect_api.read('info/version')[:data]
+          Log.dump(:connect_version,cinfo)
         rescue => e # Errno::ECONNREFUSED
           raise StandardError,"Unable to start connect after #{trynumber} try" if trynumber >= MAX_CONNECT_START_RETRY
           Log.log.warn("connect is not started. Retry ##{trynumber}, err=#{e}")
@@ -37,6 +34,9 @@ module Aspera
           sleep(SLEEP_SEC_BETWEEN_RETRY)
           retry
         end
+      end
+
+      def start_transfer(transfer_spec,options=nil)
         if transfer_spec['direction'] == 'send'
           Log.log.warn("Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red)
           transfer_spec.delete('paths')
@@ -47,7 +47,6 @@ module Aspera
         # if there is a token, we ask connect client to use well known ssh private keys
         # instead of asking password
         transfer_spec['authentication']='token' if transfer_spec.has_key?('token')
-        connect_settings=
         connect_transfer_args={
           'aspera_connect_settings'=>@connect_settings.merge({
           'request_id'               =>@request_id,
@@ -65,42 +64,50 @@ module Aspera
         connect_activity_args={'aspera_connect_settings'=>@connect_settings}
         started=false
         spinner=nil
-        loop do
-          tr_info=@connect_api.create("transfers/info/#{@xfer_id}",connect_activity_args)[:data]
-          if tr_info['transfer_info'].is_a?(Hash)
-            trdata=tr_info['transfer_info']
-            if trdata.nil?
-              Log.log.warn("no session in Connect")
-              break
-            end
-            # TODO: get session id
-            case trdata['status']
-            when 'completed'
-              notify_end(@connect_settings['app_id'])
-              break
-            when 'initiating','queued'
-              if spinner.nil?
-                spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
-                spinner.start
+        begin
+          loop do
+            tr_info=@connect_api.create("transfers/info/#{@xfer_id}",connect_activity_args)[:data]
+            if tr_info['transfer_info'].is_a?(Hash)
+              trdata=tr_info['transfer_info']
+              if trdata.nil?
+                Log.log.warn("no session in Connect")
+                break
               end
-              spinner.update(title: trdata['status'])
-              spinner.spin
-            when 'running'
-              #puts "running: sessions:#{trdata['sessions'].length}, #{trdata['sessions'].map{|i| i['bytes_transferred']}.join(',')}"
-              if !started and trdata['bytes_expected'] != 0
-                notify_begin(@connect_settings['app_id'],trdata['bytes_expected'])
-                started=true
+              # TODO: get session id
+              case trdata['status']
+              when 'completed'
+                notify_end(@connect_settings['app_id'])
+                break
+              when 'initiating','queued'
+                if spinner.nil?
+                  spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
+                  spinner.start
+                end
+                spinner.update(title: trdata['status'])
+                spinner.spin
+              when 'running'
+                #puts "running: sessions:#{trdata['sessions'].length}, #{trdata['sessions'].map{|i| i['bytes_transferred']}.join(',')}"
+                if !started and trdata['bytes_expected'] != 0
+                  spinner.success unless spinner.nil?
+                  notify_begin(@connect_settings['app_id'],trdata['bytes_expected'])
+                  started=true
+                else
+                  notify_progress(@connect_settings['app_id'],trdata['bytes_written'])
+                end
+              when 'failed'
+                spinner.error unless spinner.nil?
+                raise Fasp::Error.new(trdata['error_desc'])
               else
-                notify_progress(@connect_settings['app_id'],trdata['bytes_written'])
+                raise Fasp::Error.new("unknown status: #{trdata['status']}: #{trdata['error_desc']}")
               end
-            else
-              raise Fasp::Error.new("unknown status: #{trdata['status']}: #{trdata['error_desc']}")
             end
+            sleep 1
           end
-          sleep 1
+        rescue => e
+          return [e]
         end
-        return [] #TODO
+        return [:success]
       end # wait
-    end # Connect
+    end # AgentConnect
   end
 end

data/lib/aspera/fasp/{local.rb → agent_direct.rb}

@@ -5,11 +5,12 @@
 # Laurent Martin
 #
 ##############################################################################
-require 'aspera/fasp/manager'
+require 'aspera/fasp/agent_base'
 require 'aspera/fasp/error'
 require 'aspera/fasp/parameters'
 require 'aspera/fasp/installation'
 require 'aspera/fasp/resume_policy'
+require 'aspera/fasp/default'
 require 'aspera/log'
 require 'socket'
 require 'timeout'
@@ -17,21 +18,18 @@ require 'securerandom'
 
 module Aspera
   module Fasp
-    # (public) default transfer username for access key based transfers
-    ACCESS_KEY_TRANSFER_USER='xfer'
     # executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
-    class Local < Manager
-      # options for initialize
+    class AgentDirect < AgentBase
+      # options for initialize (same as values in option transfer_info)
       DEFAULT_OPTIONS = {
-        :spawn_timeout_sec => 3,
-        :spawn_delay_sec   => 2,
-        :wss               => false,
-        :resume            => {}
+        spawn_timeout_sec: 3,
+        spawn_delay_sec:   2,
+        wss:               false,
+        multi_incr_udp:    true,
+        resume:            {},
+        quiet:             true   # by default no interactive progress bar
       }
-      DEFAULT_UDP_PORT=33001
       private_constant :DEFAULT_OPTIONS
-      # set to false to keep ascp progress bar display ("true" adds ascp's option -q)
-      attr_accessor :quiet
 
       # start ascp transfer (non blocking), single or multi-session
       # job information added to @jobs
@@ -64,21 +62,27 @@ module Aspera
           transfer_spec['EX_ssh_key_paths'] = Installation.instance.bypass_keys
         end
 
-        # TODO: check if changing fasp(UDP) port is really necessary, not clear from doc
-        # compute this before using transfer spec, even if the var is not used in single session
-        multi_session_udp_port_base=DEFAULT_UDP_PORT
-        multi_session_number=0
+        # Compute this before using transfer spec because it potentially modifies the transfer spec
+        # (even if the var is not used in single session)
+        multi_session_info=nil
         if transfer_spec.has_key?('multi_session')
-          multi_session_number=transfer_spec['multi_session'].to_i
-          if multi_session_number < 0
+          multi_session_info={
+            count: transfer_spec['multi_session'].to_i,
+          }
+          # Managed by multi-session, so delete from transfer spec
+          transfer_spec.delete('multi_session')
+          if multi_session_info[:count] < 0
             Log.log.error("multi_session(#{transfer_spec['multi_session']}) shall be integer >= 0")
-            multi_session_number = 0
-          end
-          if multi_session_number > 0
-            # managed here, so delete from transfer spec
-            transfer_spec.delete('multi_session')
-            if transfer_spec.has_key?('fasp_port')
-              multi_session_udp_port_base=transfer_spec['fasp_port']
+            multi_session_info = nil
+          elsif multi_session_info[:count].eql?(0)
+            Log.log.debug("multi_session count is zero: no multisession")
+            multi_session_info = nil
+          else # multi_session_info[:count] > 0
+            # if option not true: keep default udp port for all sessions
+            if @options[:multi_incr_udp]
+              # override if specified, else use default value
+              multi_session_info[:udp_base]=transfer_spec.has_key?('fasp_port') ? transfer_spec['fasp_port'] : Default::UDP_PORT
+              # delete from original transfer spec, as we will increment values
               transfer_spec.delete('fasp_port')
             end
           end
@@ -93,7 +97,7 @@ module Aspera
           env_args[:args].unshift('-I',Installation.instance.path(:fallback_cert))
         end
 
-        env_args[:args].unshift('-q') if @quiet
+        env_args[:args].unshift('-q') if @options[:quiet]
 
         # transfer job can be multi session
         xfer_job={
@@ -111,22 +115,23 @@ module Aspera
           :options  => job_options  # [Hash]
         }
 
-        if multi_session_number <= 1
+        if multi_session_info.nil?
           Log.log.debug('Starting single session thread')
           # single session for transfer : simple
           session[:thread] = Thread.new(session) {|s|transfer_thread_entry(s)}
           xfer_job[:sessions].push(session)
         else
           Log.log.debug('Starting multi session threads')
-          1.upto(multi_session_number) do |i|
+          1.upto(multi_session_info[:count]) do |i|
+            # do not delay the first session
             sleep(@options[:spawn_delay_sec]) unless i.eql?(1)
             # do deep copy (each thread has its own copy because it is modified here below and in thread)
             this_session=session.clone()
             this_session[:env_args]=this_session[:env_args].clone()
             this_session[:env_args][:args]=this_session[:env_args][:args].clone()
-            this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_number}")
-            # necessary only if server is not linux, i.e. server does not support port re-use
-            this_session[:env_args][:args].unshift('-O',"#{multi_session_udp_port_base+i-1}")
+            this_session[:env_args][:args].unshift("-C#{i}:#{multi_session_info[:count]}")
+            # option: increment (default as per ascp manual) or not (cluster on other side ?)
+            this_session[:env_args][:args].unshift('-O',"#{multi_session_info[:udp_base]+i-1}") if @options[:multi_incr_udp]
             this_session[:thread] = Thread.new(this_session) {|s|transfer_thread_entry(s)}
             xfer_job[:sessions].push(this_session)
           end
@@ -234,7 +239,7 @@ module Aspera
             when ''
               # empty line is separator to end event information
               raise 'unexpected empty line' if current_event_data.nil?
-              current_event_data[Manager::LISTENER_SESSION_ID_B]=ascp_pid
+              current_event_data[AgentBase::LISTENER_SESSION_ID_B]=ascp_pid
               notify_listeners(current_event_text,current_event_data)
               case current_event_data['Type']
               when 'INIT'
@@ -260,6 +265,7 @@ module Aspera
                 Log.log.error('need to regenerate token'.red)
                 if session[:options].is_a?(Hash) and session[:options].has_key?(:regenerate_token)
                   # regenerate token here, expired, or error on it
+                  # Note: in multi-session, each session will have a different one.
                   env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
                 end
               end
@@ -323,16 +329,14 @@ module Aspera
 
       private
 
-      # @param options : keys(symbol): wss, resume
+      # @param options : keys(symbol): see DEFAULT_OPTIONS
       def initialize(options=nil)
         super()
-        # by default no interactive progress bar
-        @quiet=true
         # all transfer jobs, key = SecureRandom.uuid, protected by mutex, condvar on change
         @jobs={}
         # mutex protects global data accessed by threads
         @mutex=Mutex.new
-        # manage options
+        # set default options and override if specified
         @options=DEFAULT_OPTIONS.clone
         if !options.nil?
           raise "expecting Hash (or nil), but have #{options.class}" unless options.is_a?(Hash)
@@ -340,7 +344,7 @@ module Aspera
             if DEFAULT_OPTIONS.has_key?(k)
               @options[k]=v
             else
-              raise "unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map{|i|i.to_s}.join(",")}"
+              raise "Unknown local agent parameter: #{k}, expect one of #{DEFAULT_OPTIONS.keys.map{|i|i.to_s}.join(",")}"
             end
           end
         end
@@ -367,6 +371,6 @@ module Aspera
         Log.log.debug("EXIT (#{Thread.current[:name]})")
       end
 
-    end # Local
+    end # AgentDirect
   end
 end