aspera-cli 4.18.0 → 4.19.0

data/lib/aspera/ascp/installation.rb

@@ -41,8 +41,11 @@ module Aspera
         </CONF>
       END_OF_CONFIG_FILE
       # all ascp files (in SDK)
-      FILES = %i[ascp ascp4 transferd ssh_private_dsa ssh_private_rsa aspera_license aspera_conf fallback_certificate fallback_private_key].freeze
+      EXE_FILES = %i[ascp ascp4 async].freeze
+      FILES = %i[transferd ssh_private_dsa ssh_private_rsa aspera_license aspera_conf fallback_certificate fallback_private_key].unshift(*EXE_FILES).freeze
       private_constant :EXT_RUBY_PROTOBUF, :RB_SDK_FOLDER, :DEFAULT_ASPERA_CONF, :FILES
+      # options for SSH client private key
+      CLIENT_SSH_KEY_OPTIONS = %i{dsa_rsa rsa per_client}.freeze
       # set ascp executable path
       def ascp_path=(v)
         @path_to_ascp = v
@@ -110,14 +113,14 @@ module Aspera
       def path(k)
         file_is_optional = false
         case k
-        when :ascp, :ascp4
+        when *EXE_FILES
+          file_is_optional = k.eql?(:async)
           use_ascp_from_product(FIRST_FOUND) if @path_to_ascp.nil?
-          file = @path_to_ascp
           # NOTE: that there might be a .exe at the end
-          file = file.gsub('ascp', 'ascp4') if k.eql?(:ascp4)
+          file = @path_to_ascp.gsub('ascp', k.to_s)
         when :transferd
-          file = transferd_filepath
           file_is_optional = true
+          file = transferd_filepath
         when :ssh_private_dsa, :ssh_private_rsa
           # assume last 3 letters are type
           type = k.to_s[-3..-1].to_sym
@@ -151,8 +154,16 @@ module Aspera
         return DataRepository.instance.item(:uuid)
       end
 
-      def aspera_token_ssh_key_paths
-        return %i[ssh_private_dsa ssh_private_rsa].map{|i|Installation.instance.path(i)}
+      # get paths of SSH keys to use for ascp client
+      # @param types [Symbol] types to use
+      def aspera_token_ssh_key_paths(types)
+        Aspera.assert_values(types, CLIENT_SSH_KEY_OPTIONS)
+        return case types
+               when :dsa_rsa, :rsa
+                 types.to_s.split('_').map{|i|Installation.instance.path("ssh_private_#{i}".to_sym)}
+               when :per_client
+                 raise 'Not yet implemented'
+               end
       end
 
       # use in plugin `config`
@@ -169,13 +180,14 @@ module Aspera
         raise "An error occurred when testing #{ascp_filename}: #{cmd_out}" unless $CHILD_STATUS == 0
         # get version from ascp, only after full extract, as windows requires DLLs (SSL/TLS/etc...)
         m = cmd_out.match(/ version ([0-9.]+)/)
-        exe_version = m[1] unless m.nil?
+        exe_version = m[1].gsub(/\.$/, '') unless m.nil?
         return exe_version
       end
 
-      def ascp_add_pvcl(data)
+      def ascp_pvcl_info
+        data = {}
         # read PATHs from ascp directly, and pvcl modules as well
-        Open3.popen3(data['ascp'], '-DDL-') do |_stdin, _stdout, stderr, thread|
+        Open3.popen3(ascp_path, '-DDL-') do |_stdin, _stdout, stderr, thread|
           last_line = ''
           while (line = stderr.gets)
             line.chomp!
@@ -194,32 +206,32 @@ module Aspera
               data['product_name'] = Regexp.last_match(1)
               data['product_version'] = Regexp.last_match(2)
             when /^LOG Initializing FASP version ([^,]+),/
-              data['ascp_version'] = Regexp.last_match(1)
+              data['sdk_ascp_version'] = Regexp.last_match(1)
             end
           end
           if !thread.value.exitstatus.eql?(1) && !data.key?('root')
             raise last_line
           end
         end
+        return data
       end
 
       # extract some stings from ascp binary
-      def ascp_add_openssl(data)
-        ascp_file = data['ascp']
-        File.binread(ascp_file).scan(/[\x20-\x7E]{10,}/) do |bin_string|
+      def ascp_ssl_info
+        data = {}
+        File.binread(ascp_path).scan(/[\x20-\x7E]{10,}/) do |bin_string|
           if (m = bin_string.match(/OPENSSLDIR.*"(.*)"/))
             data['openssldir'] = m[1]
           elsif (m = bin_string.match(/OpenSSL (\d[^ -]+)/))
             data['openssl_version'] = m[1]
           end
-        end if File.file?(ascp_file)
+        end if File.file?(ascp_path)
+        return data
       end
 
       def ascp_info
-        data = file_paths
-        ascp_add_pvcl(data)
-        ascp_add_openssl(data)
-        return data
+        files = file_paths
+        return files.merge(ascp_pvcl_info).merge(ascp_ssl_info)
       end
 
       # download aspera SDK or use local file
@@ -263,19 +275,22 @@ module Aspera
         # ensure license file are generated so that ascp invocation for version works
         path(:aspera_license)
         path(:aspera_conf)
-        ascp_file = Products.ascp_filename
-        ascp_path = File.join(sdk_folder, ascp_file)
-        raise "No #{ascp_file} found in SDK archive" unless File.exist?(ascp_path)
-        Environment.restrict_file_access(ascp_path, mode: 0o755)
-        Environment.restrict_file_access(ascp_path.gsub('ascp', 'ascp4'), mode: 0o755)
-        ascp_version = get_ascp_version(ascp_path)
-        trd_path = transferd_filepath
-        Log.log.warn{"No #{trd_path} in SDK archive"} unless File.exist?(trd_path)
-        Environment.restrict_file_access(trd_path, mode: 0o755) if File.exist?(trd_path)
-        transferd_version = get_exe_version(trd_path, 'version')
-        sdk_version = transferd_version || ascp_version
-        File.write(File.join(sdk_folder, Products::INFO_META_FILE), "<product><name>IBM Aspera SDK</name><version>#{sdk_version}</version></product>")
-        return sdk_version
+        sdk_ascp_file = Products.ascp_filename
+        sdk_ascp_path = File.join(sdk_folder, sdk_ascp_file)
+        raise "No #{sdk_ascp_file} found in SDK archive" unless File.exist?(sdk_ascp_path)
+        EXE_FILES.each do |exe_sym|
+          exe_path = sdk_ascp_path.gsub('ascp', exe_sym.to_s)
+          Environment.restrict_file_access(exe_path, mode: 0o755) if File.exist?(exe_path)
+        end
+        sdk_ascp_version = get_ascp_version(sdk_ascp_path)
+        sdk_daemon_path = transferd_filepath
+        Log.log.warn{"No #{sdk_daemon_path} in SDK archive"} unless File.exist?(sdk_daemon_path)
+        Environment.restrict_file_access(sdk_daemon_path, mode: 0o755) if File.exist?(sdk_daemon_path)
+        transferd_version = get_exe_version(sdk_daemon_path, 'version')
+        sdk_name = 'IBM Aspera Transfer SDK'
+        sdk_version = transferd_version || sdk_ascp_version
+        File.write(File.join(sdk_folder, Products::INFO_META_FILE), "<product><name>#{sdk_name}</name><version>#{sdk_version}</version></product>")
+        return sdk_name, sdk_version
       end
 
       private

data/lib/aspera/ascp/management.rb

@@ -188,7 +188,7 @@ module Aspera
       # empty line is separator to end event information
       MGT_FRAME_SEPARATOR = ''
       # fields description for JSON generation
-      # spellchecker: disable
+      # cspell: disable
       INTEGER_FIELDS = %w[Bytescont FaspFileArgIndex StartByte Rate MinRate Port Priority RateCap MinRateCap TCPPort CreatePolicy TimePolicy
                           DatagramSize XoptFlags VLinkVersion PeerVLinkVersion DSPipelineDepth PeerDSPipelineDepth ReadBlockSize WriteBlockSize
                           ClusterNumNodes ClusterNodeId Size Written Loss FileBytes PreTransferBytes TransferBytes PMTU Elapsedusec ArgScansAttempted
@@ -219,6 +219,9 @@ module Aspera
       end
       attr_reader :last_event
 
+      # process line of mgt port event
+      # @param line [String] line of mgt port event
+      # @return [Hash] event hash or nil if event is not yet complete
       def process_line(line)
         # Log.log.debug{"line=[#{line}]"}
         case line

data/lib/aspera/ascp/products.rb

@@ -44,7 +44,7 @@ module Aspera
               app_root: File.join('C:', 'Program Files', 'Aspera', 'Enterprise Server'),
               log_root: File.join('C:', 'Program Files', 'Aspera', 'Enterprise Server', 'var', 'log')
             }]
-            when Aspera::Environment::OS_X then [{
+            when Aspera::Environment::OS_MACOS then [{
               expected: CONNECT,
               app_root: File.join(Dir.home, 'Applications', 'Aspera Connect.app'),
               log_root: File.join(Dir.home, 'Library', 'Logs', 'Aspera_Connect'),
@@ -126,7 +126,7 @@ module Aspera
 
         # filename for ascp with optional extension (Windows)
         def ascp_filename
-          return 'ascp' + Environment.exe_extension
+          return "ascp#{Environment.exe_extension}"
         end
 
         # @return folder paths for specified applications
@@ -150,12 +150,6 @@ module Aspera
           end
           raise "no connect uri file found in #{folder}"
         end
-
-        # @ return path to configuration file of aspera CLI
-        # def cli_conf_file
-        #  connect = folders(PRODUCT_CLI_V1)
-        #  return File.join(connect[:app_root], BIN_SUBFOLDER, '.aspera_cli_conf')
-        # end
       end
     end
   end

data/lib/aspera/cli/extended_value.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# cspell:ignore csvt jsonpp
+# cspell:ignore csvt jsonpp stdbin
 require 'aspera/uri_reader'
 require 'aspera/environment'
 require 'aspera/log'
@@ -17,11 +17,6 @@ module Aspera
     class ExtendedValue
       include Singleton
 
-      # special values
-      INIT = 'INIT'
-      ALL = 'ALL'
-      DEF = 'DEF'
-
       MARKER_START = '@'
       MARKER_END = ':'
       MARKER_IN_END = '@'
@@ -74,10 +69,22 @@ module Aspera
           zlib:   lambda{|v|Zlib::Inflate.inflate(v)},
           extend: lambda{|v|ExtendedValue.instance.evaluate_all(v)}
         }
+        @default_decoder = nil
+      end
+
+      # Regex to match an extended value
+      def handler_regex_string
+        "#{MARKER_START}(#{modifiers.join('|')})#{MARKER_END}"
       end
 
       public
 
+      def default_decoder=(value)
+        Log.log.debug{"setting default decoder to #{value} (#{value.class})"}
+        Aspera.assert(value.nil? || @handlers.key?(value))
+        @default_decoder = value
+      end
+
       def modifiers; @handlers.keys; end
 
       # add a new handler
@@ -87,16 +94,13 @@ module Aspera
         @handlers[name] = method
       end
 
-      # Regex to match an extended value
-      def ext_re
-        "#{MARKER_START}(#{modifiers.join('|')})#{MARKER_END}"
-      end
-
-      # parse an option value if it is a String using supported extended value modifiers
+      # parse an string value to extended value, if it is a String using supported extended value modifiers
       # other value types are returned as is
+      # @param value [String] the value to parse
+      # @param expect [Class,Array] one or a list of expected types
       def evaluate(value)
         return value unless value.is_a?(String)
-        regex = Regexp.new("^#{ext_re}(.*)$", Regexp::MULTILINE)
+        regex = Regexp.new("^#{handler_regex_string}(.*)$", Regexp::MULTILINE)
         # first determine decoders, in reversed order
         handlers_reversed = []
         while (m = value.match(regex))
@@ -113,9 +117,18 @@ module Aspera
         return value
       end
 
+      # parse string value as extended value
+      # use default decoder if none is specified
+      def evaluate_with_default(value)
+        if value.is_a?(String) && value.match(/^#{handler_regex_string}.*$/).nil? && !@default_decoder.nil?
+          value = [MARKER_START, @default_decoder, MARKER_END, value].join
+        end
+        return evaluate(value)
+      end
+
       # find inner extended values
       def evaluate_all(value)
-        regex = Regexp.new("^(.*)#{ext_re}([^#{MARKER_IN_END}]*)#{MARKER_IN_END}(.*)$", Regexp::MULTILINE)
+        regex = Regexp.new("^(.*)#{handler_regex_string}([^#{MARKER_IN_END}]*)#{MARKER_IN_END}(.*)$", Regexp::MULTILINE)
         while (m = value.match(regex))
           sub_value = "@#{m[2]}:#{m[3]}"
           Log.log.debug{"evaluating #{sub_value}"}

data/lib/aspera/cli/formatter.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 # cspell:ignore jsonpp
+require 'aspera/cli/special_values'
 require 'aspera/preview/terminal'
 require 'aspera/secret_hider'
 require 'aspera/environment'
@@ -97,11 +98,12 @@ module Aspera
       CSV_RECORD_SEPARATOR = "\n"
       CSV_FIELD_SEPARATOR = ','
       # supported output formats
-      DISPLAY_FORMATS = %i[text nagios ruby json jsonpp yaml table csv image].freeze
+      DISPLAY_FORMATS = %i[text nagios ruby json jsonpp yaml table multi csv image].freeze
       # user output levels
       DISPLAY_LEVELS = %i[info data error].freeze
+      FIELD_VALUE_HEADINGS = %i[key value].freeze
 
-      private_constant :DISPLAY_FORMATS, :DISPLAY_LEVELS, :CSV_RECORD_SEPARATOR, :CSV_FIELD_SEPARATOR
+      private_constant :DISPLAY_FORMATS, :DISPLAY_LEVELS, :CSV_RECORD_SEPARATOR, :CSV_FIELD_SEPARATOR, :FIELD_VALUE_HEADINGS
       # prefix to display error messages in user messages (terminal)
       ERROR_FLASH = 'ERROR:'.bg_red.gray.blink.freeze
       WARNING_FLASH = 'WARNING:'.bg_brown.black.blink.freeze
@@ -110,12 +112,12 @@ module Aspera
       class << self
         def all_but(list)
           list = [list] unless list.is_a?(Array)
-          return list.map{|i|"#{FIELDS_LESS}#{i}"}.unshift(ExtendedValue::ALL)
+          return list.map{|i|"#{FIELDS_LESS}#{i}"}.unshift(SpecialValues::ALL)
         end
 
         def tick(yes)
           result =
-            if Environment.terminal_supports_unicode?
+            if Environment.instance.terminal_supports_unicode?
               if yes
                 "\u2713"
               else
@@ -149,16 +151,9 @@ module Aspera
       end
 
       # Highlight special values
-      def special_format(what, use_colors: $stdout.isatty)
-        result = $stdout.isatty ? "<#{what}>" : "&lt;#{what}&gt;"
-        if use_colors
-          result = if %w[null empty].any?{|s|what.include?(s)}
-            result.dim
-          else
-            result.reverse_color
-          end
-        end
-        return result
+      def special_format(what)
+        result = "<#{what}>"
+        return %w[null empty].any?{|s|what.include?(s)} ? result.dim : result.reverse_color
       end
 
       # call this after REST calls if several api calls are expected
@@ -197,7 +192,7 @@ module Aspera
       end
 
       def declare_options(options)
-        default_table_style = if Environment.terminal_supports_unicode?
+        default_table_style = if Environment.instance.terminal_supports_unicode?
           {border: :unicode_round}
         else
           {}
@@ -207,9 +202,9 @@ module Aspera
         options.declare(:output, 'Destination for results', types: String, handler: {o: self, m: :option_handler})
         options.declare(:display, 'Output only some information', values: DISPLAY_LEVELS, handler: {o: self, m: :option_handler}, default: :info)
         options.declare(
-          :fields, "Comma separated list of: fields, or #{ExtendedValue::ALL}, or #{ExtendedValue::DEF}", handler: {o: self, m: :option_handler},
+          :fields, "Comma separated list of: fields, or #{SpecialValues::ALL}, or #{SpecialValues::DEF}", handler: {o: self, m: :option_handler},
           types: [String, Array, Regexp, Proc],
-          default: ExtendedValue::DEF)
+          default: SpecialValues::DEF)
         options.declare(:select, 'Select only some items in lists: column, value', types: [Hash, Proc], handler: {o: self, m: :option_handler})
         options.declare(:table_style, 'Table display style', types: [Hash], handler: {o: self, m: :option_handler}, default: default_table_style)
         options.declare(:flat_hash, 'Display deep values as additional keys', values: :bool, handler: {o: self, m: :option_handler}, default: true)
@@ -256,7 +251,7 @@ module Aspera
         # the requested list of fields, but if can contain special values
         request =
           case @options[:fields]
-          # when NilClass then [ExtendedValue::DEF]
+          # when NilClass then [SpecialValues::DEF]
           when String then @options[:fields].split(',')
           when Array then @options[:fields]
           when Regexp then return all_fields(data).select{|i|i.match(@options[:fields])}
@@ -272,10 +267,10 @@ module Aspera
             item = item[1..-1]
           end
           case item
-          when ExtendedValue::ALL
+          when SpecialValues::ALL
             # get the list of all column names used in all lines, not just first one, as all lines may have different columns
             request.unshift(*all_fields(data))
-          when ExtendedValue::DEF
+          when SpecialValues::DEF
             default = all_fields(data).select{|i|default.call(i)} if default.is_a?(Proc)
             default = all_fields(data) if default.nil?
             request.unshift(*default)
@@ -293,11 +288,11 @@ module Aspera
       # filter the list of items on the fields option
       def filter_list_on_fields(data)
         # by default, keep all data intact
-        return data if @options[:fields].eql?(ExtendedValue::DEF) && @options[:select].nil?
+        return data if @options[:fields].eql?(SpecialValues::DEF) && @options[:select].nil?
         Aspera.assert_type(data, Array){'Filtering fields or select requires result is an Array of Hash'}
         Aspera.assert(data.all?(Hash)){'Filtering fields or select requires result is an Array of Hash'}
         filter_columns_on_select(data)
-        return data if @options[:fields].eql?(ExtendedValue::DEF)
+        return data if @options[:fields].eql?(SpecialValues::DEF)
         selected_fields = compute_fields(data, @options[:fields])
         return data.map{|i|i[selected_fields.first]} if selected_fields.length == 1
         return data.map{|i|i.select{|k, _|selected_fields.include?(k)}}
@@ -325,16 +320,16 @@ module Aspera
           display_message(:info, special_format('empty')) if @options[:format].eql?(:table)
           return
         end
+        # if table has only one element, and only one field, display the value
         if object_array.length == 1 && fields.length == 1
           display_message(:data, object_array.first[fields.first])
           return
         end
         # Special case if only one row (it could be object_list or single_object)
         if @options[:transpose_single] && object_array.length == 1
-          new_columns = %i[key value]
           single = object_array.first
-          object_array = fields.map { |i| new_columns.zip([i, single[i]]).to_h }
-          fields = new_columns
+          object_array = fields.map { |i| FIELD_VALUE_HEADINGS.zip([i, single[i]]).to_h }
+          fields = FIELD_VALUE_HEADINGS
         end
         Log.log.debug{Log.dump(:object_array, object_array)}
         # convert data to string, and keep only display fields
@@ -347,8 +342,18 @@ module Aspera
             headings:  fields,
             rows:      final_table_rows,
             style:     @options[:table_style]&.symbolize_keys))
+        when :multi
+          final_table_rows.each do |row|
+            Log.log.debug{Log.dump(:row, row)}
+            display_message(:data, Terminal::Table.new(
+              headings:  FIELD_VALUE_HEADINGS,
+              rows:      fields.zip(row),
+              style:     @options[:table_style]&.symbolize_keys))
+          end
         when :csv
           display_message(:data, final_table_rows.map{|t| t.join(CSV_FIELD_SEPARATOR)}.join(CSV_RECORD_SEPARATOR))
+        else
+          raise "not expected: #{@options[:format]}"
         end
       end
 
@@ -358,8 +363,8 @@ module Aspera
           raise URI::InvalidURIError, 'not uri' if !(blob =~ /\A#{URI::DEFAULT_PARSER.make_regexp}\z/)
           # it's a url
           url = blob
-          unless OpenApplication.instance.url_method.eql?(:text)
-            OpenApplication.instance.uri(url)
+          unless Environment.instance.url_method.eql?(:text)
+            Environment.instance.open_uri(url)
             return ''
           end
           # remote_image = Rest.new(base_url: url).read('')
@@ -419,7 +424,7 @@ module Aspera
           end
           raise "not url: #{url.class} #{url}" unless url.is_a?(String)
           display_message(:data, status_image(url))
-        when :table, :csv
+        when :table, :csv, :multi
           case type
           when :config_over
             display_table(Flattener.new(self).config_over(data), CONF_OVERVIEW_KEYS)
@@ -451,6 +456,8 @@ module Aspera
           else
             raise "unknown data type: #{type}"
           end
+        else
+          raise "not expected: #{@options[:format]}"
         end
       end
     end

data/lib/aspera/cli/main.rb

@@ -93,11 +93,11 @@ module Aspera
         @plug_init[:only_manual] = false
         # create formatter, in case there is an exception, it is used to display.
         @plug_init[:formatter] = Formatter.new
-        @plug_init[:options] = Manager.new(PROGRAM_NAME)
-        # give command line arguments to option manager
-        options.parse_command_line(@argv)
+        # create command line manager with arguments
+        @plug_init[:options] = Manager.new(PROGRAM_NAME, @argv)
         # formatter adds options
-        formatter.declare_options(options)
+        @plug_init[:formatter].declare_options(options)
+        ExtendedValue.instance.default_decoder = options.get_option(:struct_parser)
         # compare $0 with expected name
         current_prog_name = File.basename($PROGRAM_NAME)
         formatter.display_message(
@@ -162,9 +162,9 @@ module Aspera
         options.declare(:version, 'Display version', values: :none, short: 'v') { formatter.display_message(:data, Cli::VERSION); Process.exit(0) } # rubocop:disable Style/Semicolon, Layout/LineLength
         options.declare(
           :ui, 'Method to start browser',
-          values: OpenApplication::USER_INTERFACES,
-          handler: {o: OpenApplication.instance, m: :url_method},
-          default: OpenApplication.default_gui_mode)
+          values: Environment::USER_INTERFACES,
+          handler: {o: Environment.instance, m: :url_method},
+          default: Environment.default_gui_mode)
         options.declare(:log_level, 'Log level', values: Log.levels, handler: {o: Log.instance, m: :level})
         options.declare(:logger, 'Logging method', values: Log::LOG_TYPES, handler: {o: Log.instance, m: :logger_type})
         options.declare(:lock_port, 'Prevent dual execution of a command, e.g. in cron', coerce: Integer, types: Integer)
@@ -189,7 +189,7 @@ module Aspera
       end
 
       def generate_bash_completion
-        if options.get_next_argument('', expected: :multiple, mandatory: false).nil?
+        if options.get_next_argument('', multiple: true, mandatory: false).nil?
           PluginFactory.instance.plugin_list.each{|p|puts p}
         else
           Log.log.warn('only first level completion so far')
@@ -197,11 +197,11 @@ module Aspera
         Process.exit(0)
       end
 
-      def exit_with_usage(all_plugins)
-        Log.log.debug('exit_with_usage'.bg_red)
+      def exit_with_usage(include_all_plugins)
+        Log.log.debug{"exit_with_usage(#{include_all_plugins})".bg_red}
         # display main plugin options
         formatter.display_message(:error, options.parser)
-        if all_plugins
+        if include_all_plugins
           # list plugins that have a "require" field, i.e. all but main plugin
           PluginFactory.instance.plugin_list.each do |plugin_name_sym|
             next if plugin_name_sym.eql?(COMMAND_CONFIG)