aspera-cli 4.12.0 → 4.14.0

data/lib/aspera/cli/transfer_agent.rb

@@ -48,38 +48,55 @@ module Aspera
         @config = config
         # command line can override transfer spec
         @transfer_spec_cmdline = {'create_dir' => true}
+        @transfer_info = {}
         # the currently selected transfer agent
         @agent = nil
         @progress_listener = Listener::ProgressMulti.new
         # source/destination pair, like "paths" of transfer spec
         @transfer_paths = nil
-        @opt_mgr.set_obj_attr(:ts, self, :option_transfer_spec)
-        @opt_mgr.add_opt_simple(:ts, "Override transfer spec values (Hash, e.g. use @json: prefix), current=#{@opt_mgr.get_option(:ts)}")
-        @opt_mgr.add_opt_simple(:to_folder, 'Destination folder for transferred files')
-        @opt_mgr.add_opt_simple(:sources, "How list of transferred files is provided (#{FILE_LIST_OPTIONS.join(',')})")
-        @opt_mgr.add_opt_list(:src_type, %i[list pair], 'Type of file list')
-        @opt_mgr.add_opt_list(:transfer, TRANSFER_AGENTS, 'Type of transfer agent')
-        @opt_mgr.add_opt_simple(:transfer_info, 'Parameters for transfer agent')
-        @opt_mgr.add_opt_list(:progress, %i[none native multi], 'Type of progress bar')
-        @opt_mgr.set_option(:transfer, :direct)
-        @opt_mgr.set_option(:src_type, :list)
-        @opt_mgr.set_option(:progress, :native) # use native ascp progress bar as it is more reliable
+        @opt_mgr.declare(:ts, 'Override transfer spec values', types: Hash, handler: {o: self, m: :option_transfer_spec})
+        @opt_mgr.declare(:to_folder, 'Destination folder for transferred files')
+        @opt_mgr.declare(:sources, "How list of transferred files is provided (#{FILE_LIST_OPTIONS.join(',')})")
+        @opt_mgr.declare(:src_type, 'Type of file list', values: %i[list pair], default: :list)
+        @opt_mgr.declare(:transfer, 'Type of transfer agent', values: TRANSFER_AGENTS, default: :direct)
+        @opt_mgr.declare(:transfer_info, 'Parameters for transfer agent', types: Hash, handler: {o: self, m: :option_transfer_info})
+        @opt_mgr.declare(:progress, 'Type of progress bar', values: %i[none native multi], default: :native)
         @opt_mgr.parse_options!
       end
 
       def option_transfer_spec; @transfer_spec_cmdline; end
 
       # multiple option are merged
-      def option_transfer_spec=(value); @transfer_spec_cmdline.merge!(value); end
+      def option_transfer_spec=(value)
+        raise 'option ts shall be a Hash' unless value.is_a?(Hash)
+        @transfer_spec_cmdline.deep_merge!(value)
+      end
 
+      # add other transfer spec parameters
       def option_transfer_spec_deep_merge(ts); @transfer_spec_cmdline.deep_merge!(ts); end
 
+      # @return [Hash] transfer spec with updated values from command line, including removed values
+      def updated_ts(transfer_spec={})
+        transfer_spec.deep_merge!(@transfer_spec_cmdline)
+        # recursively remove values that are nil (user wants to delete)
+        transfer_spec.deep_do { |hash, key, value, _unused| hash.delete(key) if value.nil?}
+        return transfer_spec
+      end
+
+      def option_transfer_info; @transfer_info; end
+
+      # multiple option are merged
+      def option_transfer_info=(value)
+        raise 'option transfer_info shall be a Hash' unless value.is_a?(Hash)
+        @transfer_info.deep_merge!(value)
+      end
+
       def agent_instance=(instance)
         @agent = instance
         @agent.add_listener(Listener::Logger.new)
         # use local progress bar if asked so, or if native and non local ascp (because only local ascp has native progress bar)
-        if @opt_mgr.get_option(:progress, is_type: :mandatory).eql?(:multi) ||
-            (@opt_mgr.get_option(:progress, is_type: :mandatory).eql?(:native) && !instance.class.to_s.eql?('Aspera::Fasp::AgentDirect'))
+        if @opt_mgr.get_option(:progress, mandatory: true).eql?(:multi) ||
+            (@opt_mgr.get_option(:progress, mandatory: true).eql?(:native) && !instance.class.to_s.eql?('Aspera::Fasp::AgentDirect'))
           @agent.add_listener(@progress_listener)
         end
       end
@@ -87,24 +104,24 @@ module Aspera
       # analyze options and create new agent if not already created or set
       def set_agent_by_options
         return nil unless @agent.nil?
-        agent_type = @opt_mgr.get_option(:transfer, is_type: :mandatory)
+        agent_type = @opt_mgr.get_option(:transfer, mandatory: true)
         # agent plugin is loaded on demand to avoid loading unnecessary dependencies
         require "aspera/fasp/agent_#{agent_type}"
         agent_options = @opt_mgr.get_option(:transfer_info)
         raise CliBadArgument, "the transfer agent configuration shall be Hash, not #{agent_options.class} (#{agent_options}), "\
-          'use either @json:<json> or @preset:<parameter set name>' unless [Hash, NilClass].include?(agent_options.class)
-        # special case
-        if agent_type.eql?(:node) && agent_options.nil?
+          'e.g. use @json:<json>' unless agent_options.is_a?(Hash)
+        # special case: use default node
+        if agent_type.eql?(:node) && agent_options.empty?
           param_set_name = @config.get_plugin_default_config_name(:node)
-          raise CliBadArgument, "No default node configured, Please specify --#{:transfer_info.to_s.tr('_', '-')}" if param_set_name.nil?
+          raise CliBadArgument, "No default node configured. Please specify #{Manager.option_name_to_line(:transfer_info)}" if param_set_name.nil?
           agent_options = @config.preset_by_name(param_set_name)
         end
-        # special case
-        if agent_type.eql?(:direct) && @opt_mgr.get_option(:progress, is_type: :mandatory).eql?(:native)
-          agent_options = {} if agent_options.nil?
+        # special case: native progress bar
+        if agent_type.eql?(:direct) && @opt_mgr.get_option(:progress, mandatory: true).eql?(:native)
           agent_options[:quiet] = false
         end
-        agent_options = agent_options.symbolize_keys if agent_options.is_a?(Hash)
+        # normalize after getting from user or default node
+        agent_options = agent_options.symbolize_keys
         # get agent instance
         new_agent = Kernel.const_get("Aspera::Fasp::Agent#{agent_type.capitalize}").new(agent_options)
         self.agent_instance = new_agent
@@ -129,6 +146,7 @@ module Aspera
         return dest_folder
       end
 
+      # @return [Array] list of source files
       def source_list
         return ts_source_paths.map do |i|
           i['source']
@@ -156,7 +174,7 @@ module Aspera
         when FILE_LIST_FROM_TRANSFER_SPEC
           Log.log.debug('assume list provided in transfer spec')
           special_case_direct_with_list =
-            @opt_mgr.get_option(:transfer, is_type: :mandatory).eql?(:direct) &&
+            @opt_mgr.get_option(:transfer, mandatory: true).eql?(:direct) &&
             Fasp::Parameters.ts_has_ascp_file_list(@transfer_spec_cmdline, @opt_mgr.get_option(:transfer_info))
           raise CliBadArgument, 'transfer spec on command line must have sources' if @transfer_paths.nil? && !special_case_direct_with_list
           # here we assume check of sources is made in transfer agent
@@ -171,7 +189,7 @@ module Aspera
         if !@transfer_paths.nil?
           Log.log.warn('--sources overrides paths from --ts')
         end
-        case @opt_mgr.get_option(:src_type, is_type: :mandatory)
+        case @opt_mgr.get_option(:src_type, mandatory: true)
         when :list
           # when providing a list, just specify source
           @transfer_paths = file_list.map{|i|{'source' => i}}
@@ -196,7 +214,7 @@ module Aspera
           # init default if required in any case
           @transfer_spec_cmdline['destination_root'] ||= destination_folder(transfer_spec['direction'])
         when Fasp::TransferSpec::DIRECTION_SEND
-          if transfer_spec.dig('tags', 'aspera', 'node', 'access_key')
+          if transfer_spec.dig('tags', Fasp::TransferSpec::TAG_RESERVED, 'node', 'access_key')
             # gen4
             @transfer_spec_cmdline.delete('destination_root') if @transfer_spec_cmdline.key?('destination_root_id')
           elsif transfer_spec.key?('token')
@@ -211,14 +229,13 @@ module Aspera
         end
         # update command line paths, unless destination already has one
         @transfer_spec_cmdline['paths'] = transfer_spec['paths'] || ts_source_paths
-        transfer_spec.merge!(@transfer_spec_cmdline)
-        # remove values that are nil (user wants to delete)
-        transfer_spec.delete_if { |_key, value| value.nil? }
+        # updated transfer spec with command line
+        updated_ts(transfer_spec)
         # create transfer agent
         set_agent_by_options
         Log.log.debug{"transfer agent is a #{@agent.class}"}
         @agent.start_transfer(transfer_spec, token_regenerator: rest_token)
-        # list of : :success or error message
+        # list of: :success or "error message string"
         result = @agent.wait_for_transfers_completion
         @progress_listener.reset
         Fasp::AgentBase.validate_status_list(result)

data/lib/aspera/cli/version.rb

@@ -3,6 +3,7 @@
 module Aspera
   module Cli
     # for beta add extension : .beta1
-    VERSION = '4.12.0'
+    # for dev version add extension : .pre
+    VERSION = '4.14.0'
   end
 end

data/lib/aspera/colors.rb

@@ -1,16 +1,18 @@
 # frozen_string_literal: true
 
+# cspell:words
+
 # simple vt100 colors
 class String
   class << self
     private
 
-    def vtcmd(code); "\e[#{code}m"; end
+    def vt_cmd(code); "\e[#{code}m"; end
   end
   # see https://en.wikipedia.org/wiki/ANSI_escape_code
   # symbol is the method name added to String
   # it adds control chars to set color (and reset at the end).
-  VTSTYLES = {
+  VT_STYLES = {
     bold:          1,
     italic:        3,
     underline:     4,
@@ -33,17 +35,17 @@ class String
     bg_cyan:       46,
     bg_gray:       47
   }.freeze
-  private_constant :VTSTYLES
-  # defines methods to String, one per entry in VTSTYLES
-  VTSTYLES.each do |name, code|
+  private_constant :VT_STYLES
+  # defines methods to String, one per entry in VT_STYLES
+  VT_STYLES.each do |name, code|
     if $stderr.tty?
-      begin_seq = vtcmd(code)
+      begin_seq = vt_cmd(code)
       end_code = 0 # by default reset all
       if code <= 7 then code + 20
       elsif code <= 37 then 39
       elsif code <= 47 then 49
       end
-      end_seq = vtcmd(end_code)
+      end_seq = vt_cmd(end_code)
       define_method(name){"#{begin_seq}#{self}#{end_seq}"}
     else
       define_method(name){self}

data/lib/aspera/command_line_builder.rb

@@ -9,7 +9,7 @@ module Aspera
     # parameter with one of those tags is a command line option with --
     CLI_OPTION_TYPE_SWITCH = %i[opt_without_arg opt_with_arg].freeze
     CLI_OPTION_TYPES = %i[special ignore envvar].concat(CLI_OPTION_TYPE_SWITCH).freeze
-    OPTIONS_KEYS = %i[desc accepted_types default enum agents required cli ts].freeze
+    OPTIONS_KEYS = %i[desc accepted_types default enum agents required cli ts deprecation].freeze
     CLI_KEYS = %i[type switch convert variable].freeze
 
     private_constant :CLI_OPTION_TYPE_SWITCH, :OPTIONS_KEYS, :CLI_KEYS
@@ -37,6 +37,7 @@ module Aspera
           # by default : optional
           options[:mandatory] ||= false
           options[:desc] ||= ''
+          options[:desc] = "DEPRECATED: #{options[:deprecation]}\n#{options[:desc]}" if options.key?(:deprecation)
           cli = options[:cli]
           unsupported_cli_keys = cli.keys - CLI_KEYS
           raise "Unsupported cli keys: #{unsupported_cli_keys}" unless unsupported_cli_keys.empty?

data/lib/aspera/cos_node.rb

@@ -65,7 +65,7 @@ module Aspera
             type:     :basic,
             username: ats_info['AccessKey']['Id'],
             password: ats_info['AccessKey']['Secret']}},
-        add_tspec: {'tags'=>{'aspera'=>{'node'=>{'storage_credentials'=>@storage_credentials}}}})
+        add_tspec: {'tags'=>{Fasp::TransferSpec::TAG_RESERVED=>{'node'=>{'storage_credentials'=>@storage_credentials}}}})
       # update storage_credentials AND Rest params
       generate_token
     end

data/lib/aspera/environment.rb

@@ -3,6 +3,8 @@
 require 'aspera/log'
 require 'rbconfig'
 
+# cspell:words MEBI mswin bccwin
+
 module Aspera
   # detect OS, architecture, and specific stuff
   class Environment
@@ -84,12 +86,14 @@ module Aspera
       end
 
       # value is provided in block
-      def write_file_restricted(path, force: false)
+      def write_file_restricted(path, force: false, mode: nil)
         raise 'coding error, missing content block' unless block_given?
         if force || !File.exist?(path)
-          File.unlink(path) rescue nil # Windows may give error
+          # Windows may give error
+          File.unlink(path) rescue nil
+          # content provided by block
           File.write(path, yield)
-          restrict_file_access(path)
+          restrict_file_access(path, mode: mode)
         end
         return path
       end

data/lib/aspera/fasp/agent_connect.rb

@@ -9,7 +9,9 @@ require 'tty-spinner'
 module Aspera
   module Fasp
     class AgentConnect < Aspera::Fasp::AgentBase
+      # try twice the main init url in sequence
       CONNECT_START_URIS = ['fasp://initialize', 'fasp://initialize', 'aspera-drive://initialize', 'https://test-connect.ibmaspera.com/']
+      # delay between each try to start connect
       SLEEP_SEC_BETWEEN_RETRY = 3
       private_constant :CONNECT_START_URIS, :SLEEP_SEC_BETWEEN_RETRY
       def initialize(_options)
@@ -66,7 +68,7 @@ module Aspera
           }]}
         # asynchronous anyway
         res = @connect_api.create('transfers/start', connect_transfer_args)[:data]
-        @xfer_id = res['transfer_specs'].first['transfer_spec']['tags']['aspera']['xfer_id']
+        @xfer_id = res['transfer_specs'].first['transfer_spec']['tags'][Fasp::TransferSpec::TAG_RESERVED]['xfer_id']
       end
 
       def wait_for_transfers_completion
@@ -106,6 +108,9 @@ module Aspera
               when 'failed'
                 spinner&.error
                 raise Fasp::Error, transfer['error_desc']
+              when 'cancelled'
+                spinner&.error
+                raise Fasp::Error, 'Transfer cancelled by user'
               else
                 raise Fasp::Error, "unknown status: #{transfer['status']}: #{transfer['error_desc']}"
               end

data/lib/aspera/fasp/agent_direct.rb

@@ -25,7 +25,7 @@ module Aspera
         multi_incr_udp:    true,
         resume:            {},
         ascp_args:         [],
-        quiet:             true # by default no interactive progress bar
+        quiet:             true # by default no native ascp progress bar
       }.freeze
       private_constant :DEFAULT_OPTIONS
 
@@ -37,15 +37,15 @@ module Aspera
         # clone transfer spec because we modify it (first level keys)
         transfer_spec = transfer_spec.clone
         # if there is aspera tags
-        if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags']['aspera'].is_a?(Hash)
+        if transfer_spec['tags'].is_a?(Hash) && transfer_spec['tags'][Fasp::TransferSpec::TAG_RESERVED].is_a?(Hash)
           # TODO: what is this for ? only on local ascp ?
           # NOTE: important: transfer id must be unique: generate random id
           # using a non unique id results in discard of tags in AoC, and a package is never finalized
           # all sessions in a multi-session transfer must have the same xfer_id (see admin manual)
-          transfer_spec['tags']['aspera']['xfer_id'] ||= SecureRandom.uuid
+          transfer_spec['tags'][Fasp::TransferSpec::TAG_RESERVED]['xfer_id'] ||= SecureRandom.uuid
           Log.log.debug{"xfer id=#{transfer_spec['xfer_id']}"}
           # TODO: useful ? node only ?
-          transfer_spec['tags']['aspera']['xfer_retry'] ||= 3600
+          transfer_spec['tags'][Fasp::TransferSpec::TAG_RESERVED]['xfer_retry'] ||= 3600
         end
         Log.dump('ts', transfer_spec)
 
@@ -82,12 +82,12 @@ module Aspera
         end
 
         # compute known args
-        env_args = Parameters.ts_to_env_args(transfer_spec, wss: @options[:wss], ascp_args: @options[:ascp_args])
+        env_args =  Parameters.new(transfer_spec, @options).ascp_args
 
         # add fallback cert and key as arguments if needed
-        if %w[1 force].include?(transfer_spec['http_fallback'])
-          env_args[:args].unshift('-Y', Installation.instance.path(:fallback_key))
-          env_args[:args].unshift('-I', Installation.instance.path(:fallback_cert))
+        if ['1', 1, true, 'force'].include?(transfer_spec['http_fallback'])
+          env_args[:args].unshift('-Y', Installation.instance.path(:fallback_cert_privkey))
+          env_args[:args].unshift('-I', Installation.instance.path(:fallback_certificate))
         end
 
         env_args[:args].unshift('-q') if @options[:quiet]
@@ -183,20 +183,20 @@ module Aspera
           end
           # (optional) check it exists
           raise Fasp::Error, "no such file: #{ascp_path}" unless File.exist?(ascp_path)
-          # open random local TCP port for listening for ascp management
+          # open an available (0) local TCP port as ascp management
           mgt_sock = TCPServer.new('127.0.0.1', 0)
           # clone arguments as we eed to modify with mgt port
           ascp_arguments = env_args[:args].clone
-          # add management port
+          # add management port on the selected local port
           ascp_arguments.unshift('-M', mgt_sock.addr[1].to_s)
           # start ascp in sub process
           Log.log.debug do
-            'execute: ' +
-              env_args[:env].map{|k, v| "#{k}=#{Shellwords.shellescape(v)}"}.join(' ') +
-              ' ' +
-              Shellwords.shellescape(ascp_path) +
-              ' ' +
-              ascp_arguments.map{|a|Shellwords.shellescape(a)}.join(' ')
+            [
+              'execute:',
+              env_args[:env].map{|k, v| "#{k}=#{Shellwords.shellescape(v)}"},
+              Shellwords.shellescape(ascp_path),
+              ascp_arguments.map{|a|Shellwords.shellescape(a)}
+            ].flatten.join(' ')
           end
           # start process
           ascp_pid = Process.spawn(env_args[:env], [ascp_path, ascp_path], *ascp_arguments)
@@ -334,7 +334,7 @@ module Aspera
       # @param options : keys(symbol): see DEFAULT_OPTIONS
       def initialize(options=nil)
         super()
-        # all transfer jobs, key = SecureRandom.uuid, protected by mutex, condvar on change
+        # all transfer jobs, key = SecureRandom.uuid, protected by mutex, cond var on change
         @jobs = {}
         # mutex protects global data accessed by threads
         @mutex = Mutex.new